Tensor2Tensor中英翻译代码阅读(二)

PROBLEM = 'translate_enzh_wmt8k' # registry.list_models() # Show all registered models
MODEL = 'transformer' # Hyperparameters for the model by default
# start with "transformer_base" or 'transformer_base_single_gpu'
# if training on a single GPU
HPARAMS = 'transformer_base_single_gpu'

t2t_problem = problems.problem(PROBLEM)
t2t_problem.generate_data(DATA_DIR, TMP_DIR)

def generate_data(self, data_dir, tmp_dir, task_id=-1):
  # self.training_filepaths是一个函数，调用后会得到训练数据的路径。
  filepath_fns = {
      problem.DatasetSplit.TRAIN: self.training_filepaths,
      problem.DatasetSplit.EVAL: self.dev_filepaths,
      problem.DatasetSplit.TEST: self.test_filepaths,
  }
  
  # 如果是上面的例子，self.dataset_splits被子类重写了。
  split_paths = [(split["split"], filepath_fns[split["split"]](
      data_dir, split["shards"], shuffled=self.already_shuffled))
                 for split in self.dataset_splits]
  all_paths = []
  for _, paths in split_paths:
    all_paths.extend(paths)

  if self.is_generate_per_split:
    # 这里的例子是走这条路径
    for split, paths in split_paths:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir, split), paths)
  else:
    generator_utils.generate_files(
        self.generate_encoded_samples(
            data_dir, tmp_dir, problem.DatasetSplit.TRAIN), all_paths)

  generator_utils.shuffle_dataset(all_paths, extra_fn=self._pack_fn())

def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
  train = dataset_split == problem.DatasetSplit.TRAIN
  train_dataset = self.get_training_dataset(tmp_dir)
  datasets = train_dataset if train else _NC_TEST_DATASETS
  source_datasets = [[item[0], [item[1][0]]] for item in train_dataset]
  target_datasets = [[item[0], [item[1][1]]] for item in train_dataset]
  source_vocab = generator_utils.get_or_generate_vocab(
      data_dir,
      tmp_dir,
      self.source_vocab_name,
      self.approx_vocab_size,
      source_datasets,
      file_byte_budget=1e8,
      max_subtoken_length=self.max_subtoken_length)
  target_vocab = generator_utils.get_or_generate_vocab(
      data_dir,
      tmp_dir,
      self.target_vocab_name,
      self.approx_vocab_size,
      target_datasets,
      file_byte_budget=1e8,
      max_subtoken_length=self.max_subtoken_length)
  tag = "train" if train else "dev"
  filename_base = "wmt_enzh_%sk_tok_%s" % (self.approx_vocab_size, tag)
  data_path = translate.compile_data(tmp_dir, datasets, filename_base)
  return text_problems.text2text_generate_encoded(
      text_problems.text2text_txt_iterator(data_path + ".lang1",
                                           data_path + ".lang2"),
      source_vocab, target_vocab)

def get_or_generate_vocab(data_dir, tmp_dir, vocab_filename, vocab_size,
                          sources, file_byte_budget=1e6,
                          max_subtoken_length=None):
  """Generate a vocabulary from the datasets in sources."""

  vocab_generator = generate_lines_for_vocab(tmp_dir, sources, file_byte_budget)
  return get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                     vocab_generator, max_subtoken_length)

def generate_lines_for_vocab(tmp_dir, sources, file_byte_budget=1e6):
  """Generate lines for vocabulary generation."""
  tf.logging.info("Generating vocab from: %s", str(sources))
  for source in sources:
    url = source[0]
    filename = os.path.basename(url)
    compressed_file = maybe_download(tmp_dir, filename, url)

    for lang_file in source[1]:
      tf.logging.info("Reading file: %s" % lang_file)
      filepath = os.path.join(tmp_dir, lang_file)

      # Extract from tar if needed.
      if not tf.gfile.Exists(filepath):
        read_type = "r:gz" if filename.endswith("tgz") else "r"
        with tarfile.open(compressed_file, read_type) as corpus_tar:
          corpus_tar.extractall(tmp_dir)

      # For some datasets a second extraction is necessary.
      if lang_file.endswith(".gz"):
        new_filepath = os.path.join(tmp_dir, lang_file[:-3])
        if tf.gfile.Exists(new_filepath):
          tf.logging.info(
              "Subdirectory %s already exists, skipping unpacking" % filepath)
        else:
          tf.logging.info("Unpacking subdirectory %s" % filepath)
          gunzip_file(filepath, new_filepath)
        filepath = new_filepath

      with tf.gfile.GFile(filepath, mode="r") as source_file:
        file_byte_budget_ = file_byte_budget
        counter = 0
        countermax = int(source_file.size() / file_byte_budget_ / 2)
        for line in source_file:
          if counter < countermax:
            counter += 1
          else:
            if file_byte_budget_ <= 0:
              break
            line = line.strip()
            file_byte_budget_ -= len(line)
            counter = 0
            yield line

counter = 0
countermax = int(source_file.size() / file_byte_budget_ / 2)
for line in source_file:
  if counter < countermax:
    counter += 1
  else:
    if file_byte_budget_ <= 0:
      break
    line = line.strip()
    file_byte_budget_ -= len(line)
    counter = 0
    yield line

def get_or_generate_vocab_inner(data_dir, vocab_filename, vocab_size,
                                generator, max_subtoken_length=None,
                                reserved_tokens=None):
  """Inner implementation for vocab generators.

  Args:
    data_dir: 训练数据(data)和词典(vocab)文件的存储目录。如果空，则不保存。这里是/home/lili/t2tcn2/data
    vocab_filename: 词典文件的名字，这里是'vocab.translate_enzh_wmt8k.8192.subwords.en'
    vocab_size: 期望构造出来的词典大小，算法只能尽力构造这么大的词典，但不会完全相等，这里是8192 
    generator: 生成词典的generator，前面介绍过的代码
    max_subtoken_length: 最大的subtoken的长度。本函数和这个参数成平方关系。
    reserved_tokens: 保留token列表。`text_encoder.RESERVED_TOKENS`必须是`reserved_tokens`的前缀。如果`None`, 默认是`RESERVED_TOKENS`.

  Returns:
    A SubwordTextEncoder vocabulary object.
  """
  if data_dir and vocab_filename:
    vocab_filepath = os.path.join(data_dir, vocab_filename)
    if tf.gfile.Exists(vocab_filepath):
      tf.logging.info("Found vocab file: %s", vocab_filepath)
      return text_encoder.SubwordTextEncoder(vocab_filepath)
  else:
    vocab_filepath = None

  tf.logging.info("Generating vocab file: %s", vocab_filepath)
  vocab = text_encoder.SubwordTextEncoder.build_from_generator(
      generator, vocab_size, max_subtoken_length=max_subtoken_length,
      reserved_tokens=reserved_tokens)

  if vocab_filepath:
    tf.gfile.MakeDirs(data_dir)
    vocab.store_to_file(vocab_filepath)

  return vocab

@classmethod
def build_from_generator(cls,
                         generator,
                         target_size,
                         max_subtoken_length=None,
                         reserved_tokens=None):
  """Builds a SubwordTextEncoder from the generated text.

  Args:
    generator: yields text.
    target_size: int, approximate vocabulary size to create.
    max_subtoken_length: Maximum length of a subtoken. If this is not set,
      then the runtime and memory use of creating the vocab is quadratic in
      the length of the longest token. If this is set, then it is instead
      O(max_subtoken_length * length of longest token).
    reserved_tokens: List of reserved tokens. The global variable
      `RESERVED_TOKENS` must be a prefix of `reserved_tokens`. If this
      argument is `None`, it will use `RESERVED_TOKENS`.

  Returns:
    SubwordTextEncoder with `vocab_size` approximately `target_size`.
  """
  token_counts = collections.defaultdict(int)
  for item in generator:
    for tok in tokenizer.encode(native_to_unicode(item)):
      token_counts[tok] += 1
  encoder = cls.build_to_target_size(
      target_size, token_counts, 1, 1e3,
      max_subtoken_length=max_subtoken_length,
      reserved_tokens=reserved_tokens)
  return encoder

def encode(text):
  """Encode a unicode string as a list of tokens.

  Args:
    text: a unicode string
  Returns:
    a list of tokens as Unicode strings
  """
  if not text:
    return []
  ret = []
  token_start = 0
  # Classify each character in the input string
  is_alnum = [c in _ALPHANUMERIC_CHAR_SET for c in text]
  for pos in range(1, len(text)):
    if is_alnum[pos] != is_alnum[pos - 1]:
      token = text[token_start:pos]
      if token != u" " or token_start == 0:
        ret.append(token)
      token_start = pos
  final_token = text[token_start:]
  ret.append(final_token)
  return ret

_ALPHANUMERIC_CHAR_SET = set(
    six.unichr(i) for i in range(sys.maxunicode)
    if (unicodedata.category(six.unichr(i)).startswith("L") or
        unicodedata.category(six.unichr(i)).startswith("N")))

@classmethod
def build_to_target_size(cls,
                         target_size,
                         token_counts,
                         min_val,
                         max_val,
                         max_subtoken_length=None,
                         reserved_tokens=None,
                         num_iterations=4):
  """Builds a SubwordTextEncoder that has `vocab_size` near `target_size`.

  Uses simple recursive binary search to find a minimum token count that most
  closely matches the `target_size`.

  Args:
    target_size: Desired vocab_size to approximate.
    token_counts: A dictionary of token counts, mapping string to int.
    min_val: An integer; lower bound for the minimum token count.
    max_val: An integer; upper bound for the minimum token count.
    max_subtoken_length: Maximum length of a subtoken. If this is not set,
      then the runtime and memory use of creating the vocab is quadratic in
      the length of the longest token. If this is set, then it is instead
      O(max_subtoken_length * length of longest token).
    reserved_tokens: List of reserved tokens. The global variable
      `RESERVED_TOKENS` must be a prefix of `reserved_tokens`. If this
      argument is `None`, it will use `RESERVED_TOKENS`.
    num_iterations: An integer; how many iterations of refinement.

  Returns:
    A SubwordTextEncoder instance.

  Raises:
    ValueError: If `min_val` is greater than `max_val`.
  """
  if min_val > max_val:
    raise ValueError("Lower bound for the minimum token count "
                     "is greater than the upper bound.")
  if target_size < 1:
    raise ValueError("Target size must be positive.")

  if reserved_tokens is None:
    reserved_tokens = RESERVED_TOKENS

  def bisect(min_val, max_val):
    """Bisection to find the right size."""
    present_count = (max_val + min_val) // 2
    tf.logging.info("Trying min_count %d" % present_count)
    subtokenizer = cls()
    subtokenizer.build_from_token_counts(
        token_counts, present_count, num_iterations,
        max_subtoken_length=max_subtoken_length,
        reserved_tokens=reserved_tokens)

    # Being within 1% of the target size is ok.
    is_ok = abs(subtokenizer.vocab_size - target_size) * 100 < target_size
    # If min_val == max_val, we can't do any better than this.
    if is_ok or min_val >= max_val or present_count < 2:
      return subtokenizer

    if subtokenizer.vocab_size > target_size:
      other_subtokenizer = bisect(present_count + 1, max_val)
    else:
      other_subtokenizer = bisect(min_val, present_count - 1)

    if other_subtokenizer is None:
      return subtokenizer

    if (abs(other_subtokenizer.vocab_size - target_size) <
        abs(subtokenizer.vocab_size - target_size)):
      return other_subtokenizer
    return subtokenizer

  return bisect(min_val, max_val)

def build_from_token_counts(self,
                            token_counts,
                            min_count,
                            num_iterations=4,
                            reserved_tokens=None,
                            max_subtoken_length=None):
  """Train a SubwordTextEncoder based on a dictionary of word counts.

  Args:
    token_counts: a dictionary of Unicode strings to int.
    min_count: an integer - discard subtokens with lower counts.
    num_iterations: an integer.  how many iterations of refinement.
    reserved_tokens: List of reserved tokens. The global variable
      `RESERVED_TOKENS` must be a prefix of `reserved_tokens`. If this
      argument is `None`, it will use `RESERVED_TOKENS`.
    max_subtoken_length: Maximum length of a subtoken. If this is not set,
      then the runtime and memory use of creating the vocab is quadratic in
      the length of the longest token. If this is set, then it is instead
      O(max_subtoken_length * length of longest token).

  Raises:
    ValueError: if reserved is not 0 or len(RESERVED_TOKENS). In this case, it
      is not clear what the space is being reserved for, or when it will be
      filled in.
  """
  if reserved_tokens is None:
    reserved_tokens = RESERVED_TOKENS
  else:
    # There is not complete freedom in replacing RESERVED_TOKENS.
    for default, proposed in zip(RESERVED_TOKENS, reserved_tokens):
      if default != proposed:
        raise ValueError("RESERVED_TOKENS must be a prefix of "
                         "reserved_tokens.")

  # Initialize the alphabet. Note, this must include reserved tokens or it can
  # result in encoding failures.
  alphabet_tokens = chain(six.iterkeys(token_counts),
                          [native_to_unicode(t) for t in reserved_tokens])

  self._init_alphabet_from_tokens(alphabet_tokens)

  # Bootstrap the initial list of subtokens with the characters from the
  # alphabet plus the escaping characters.
  self._init_subtokens_from_list(list(self._alphabet),
                                 reserved_tokens=reserved_tokens)

  # We build iteratively.  On each iteration, we segment all the words,
  # then count the resulting potential subtokens, keeping the ones
  # with high enough counts for our new vocabulary.
  if min_count < 1:
    min_count = 1
  for i in range(num_iterations):
    tf.logging.info("Iteration {0}".format(i))

    # Collect all substrings of the encoded token that break along current
    # subtoken boundaries.
    subtoken_counts = collections.defaultdict(int)
    for token, count in six.iteritems(token_counts):
      iter_start_time = time.time()
      escaped_token = _escape_token(token, self._alphabet)
      subtokens = self._escaped_token_to_subtoken_strings(escaped_token)
      start = 0
      for subtoken in subtokens:
        last_position = len(escaped_token) + 1
        if max_subtoken_length is not None:
          last_position = min(last_position, start + max_subtoken_length)

        for end in range(start + 1, last_position):
          new_subtoken = escaped_token[start:end]
          subtoken_counts[new_subtoken] += count
        start += len(subtoken)
      iter_time_secs = time.time() - iter_start_time
      if iter_time_secs > 0.1:
        tf.logging.info(u"Processing token [{0}] took {1} seconds, consider "
                        "setting Text2TextProblem.max_subtoken_length to a "
                        "smaller value.".format(token, iter_time_secs))

    # Array of sets of candidate subtoken strings, by length.
    len_to_subtoken_strings = []
    for subtoken_string, count in six.iteritems(subtoken_counts):
      lsub = len(subtoken_string)
      if count >= min_count:
        while len(len_to_subtoken_strings) <= lsub:
          len_to_subtoken_strings.append(set())
        len_to_subtoken_strings[lsub].add(subtoken_string)

    # Consider the candidates longest to shortest, so that if we accept
    # a longer subtoken string, we can decrement the counts of its prefixes.
    new_subtoken_strings = []
    for lsub in range(len(len_to_subtoken_strings) - 1, 0, -1):
      subtoken_strings = len_to_subtoken_strings[lsub]
      for subtoken_string in subtoken_strings:
        count = subtoken_counts[subtoken_string]
        if count >= min_count:
          # Exclude alphabet tokens here, as they must be included later,
          # explicitly, regardless of count.
          if subtoken_string not in self._alphabet:
            new_subtoken_strings.append((count, subtoken_string))
          for l in range(1, lsub):
            subtoken_counts[subtoken_string[:l]] -= count

    # Include the alphabet explicitly to guarantee all strings are encodable.
    new_subtoken_strings.extend((subtoken_counts.get(a, 0), a)
                                for a in self._alphabet)
    new_subtoken_strings.sort(reverse=True)

    # Reinitialize to the candidate vocabulary.
    new_subtoken_strings = [subtoken for _, subtoken in new_subtoken_strings]
    if reserved_tokens:
      escaped_reserved_tokens = [
          _escape_token(native_to_unicode(t), self._alphabet)
          for t in reserved_tokens
      ]
      new_subtoken_strings = escaped_reserved_tokens + new_subtoken_strings

    self._init_subtokens_from_list(new_subtoken_strings)
    tf.logging.info("vocab_size = %d" % self.vocab_size)

self._init_alphabet_from_tokens(alphabet_tokens)

def _init_alphabet_from_tokens(self, tokens):
    """Initialize alphabet from an iterable of token or subtoken strings."""
    # Include all characters from all tokens in the alphabet to guarantee that
    # any token can be encoded. Additionally, include all escaping characters.
    self._alphabet = {c for token in tokens for c in token}
    self._alphabet |= _ESCAPE_CHARS

# Regular expression for unescaping token strings.
# '\u' is converted to '_'
# '\\' is converted to '\'
# '\213;' is converted to unichr(213)
_UNESCAPE_REGEX = re.compile(r"\\u|\\\\|\\([0-9]+);")
_ESCAPE_CHARS = set(u"\\_u;0123456789")

def _init_subtokens_from_list(self, subtoken_strings, reserved_tokens=None):
  """Initialize token information from a list of subtoken strings.

  Args:
    subtoken_strings: a list of subtokens
    reserved_tokens: List of reserved tokens. We must have `reserved_tokens`
      as None or the empty list, or else the global variable `RESERVED_TOKENS`
      must be a prefix of `reserved_tokens`.

  Raises:
    ValueError: if reserved is not 0 or len(RESERVED_TOKENS). In this case, it
      is not clear what the space is being reserved for, or when it will be
      filled in.
  """
  if reserved_tokens is None:
    reserved_tokens = []

  if reserved_tokens:
    self._all_subtoken_strings = reserved_tokens + subtoken_strings
  else:
    self._all_subtoken_strings = subtoken_strings

  # we remember the maximum length of any subtoken to avoid having to
  # check arbitrarily long strings.
  self._max_subtoken_len = max([len(s) for s in subtoken_strings])
  self._subtoken_string_to_id = {
      s: i + len(reserved_tokens)
      for i, s in enumerate(subtoken_strings) if s
  }
  # Initialize the cache to empty.
  self._cache_size = 2 ** 20
  self._cache = [(None, None)] * self._cache_size

self._subtoken_string_to_id = {
    s: i + len(reserved_tokens)
    for i, s in enumerate(subtoken_strings) if s
}

def _escape_token(token, alphabet):
  """Escape away underscores and OOV characters and append '_'.

  This allows the token to be expressed as the concatenation of a list
  of subtokens from the vocabulary. The underscore acts as a sentinel
  which allows us to invertibly concatenate multiple such lists.

  Args:
    token: A unicode string to be escaped.
    alphabet: A set of all characters in the vocabulary's alphabet.

  Returns:
    escaped_token: An escaped unicode string.

  Raises:
    ValueError: If the provided token is not unicode.
  """
  if not isinstance(token, six.text_type):
    raise ValueError("Expected string type for token, got %s" % type(token))

  token = token.replace(u"\\", u"\\\\").replace(u"_", u"\\u")
  ret = [c if c in alphabet and c != u"\n" else r"\%d;" % ord(c) for c in token]
  return u"".join(ret) + "_"

import six
import re

_UNESCAPE_REGEX = re.compile(r"\\u|\\\\|\\([0-9]+);")
_ESCAPE_CHARS = set(u"\\_u;0123456789")
def _escape_token(token, alphabet):
  """Escape away underscores and OOV characters and append '_'.

  This allows the token to be expressed as the concatenation of a list
  of subtokens from the vocabulary. The underscore acts as a sentinel
  which allows us to invertibly concatenate multiple such lists.

  Args:
    token: A unicode string to be escaped.
    alphabet: A set of all characters in the vocabulary's alphabet.

  Returns:
    escaped_token: An escaped unicode string.

  Raises:
    ValueError: If the provided token is not unicode.
  """
  if not isinstance(token, six.text_type):
    raise ValueError("Expected string type for token, got %s" % type(token))

  token = token.replace(u"\\", u"\\\\").replace(u"_", u"\\u")
  ret = [c if c in alphabet and c != u"\n" else r"\%d;" % ord(c) for c in token]
  return u"".join(ret) + "_"


def _unescape_token(escaped_token):
  """Inverse of _escape_token().

  Args:
    escaped_token: a unicode string

  Returns:
    token: a unicode string
  """

  def match(m):
    if m.group(1) is None:
      return u"_" if m.group(0) == u"\\u" else u"\\"

    try:
      return six.unichr(int(m.group(1)))
    except (ValueError, OverflowError) as _:
      return u"\u3013"  # Unicode for undefined character.

  trimmed = escaped_token[:-1] if escaped_token.endswith("_") else escaped_token
  return _UNESCAPE_REGEX.sub(match, trimmed)

tokens=["abcdefg","hijklmn","1234567890"]
_alphabet = {c for token in tokens for c in token}
_alphabet |= _ESCAPE_CHARS

for s in ["abc", "\\a\U0001F929\nc"]:
    print(s)
    escaped = _escape_token(s, _alphabet)
    print(escaped)
    unescaped = _unescape_token(escaped)
    print(unescaped)

def _escaped_token_to_subtoken_strings(self, escaped_token):
  """Converts an escaped token string to a list of subtoken strings.

  Args:
    escaped_token: An escaped token as a unicode string.
  Returns:
    A list of subtokens as unicode strings.
  """
  # NOTE: This algorithm is greedy; it won't necessarily produce the "best"
  # list of subtokens.
  ret = []
  start = 0
  token_len = len(escaped_token)
  while start < token_len:
    for end in range(
        min(token_len, start + self._max_subtoken_len), start, -1):
      subtoken = escaped_token[start:end]
      if subtoken in self._subtoken_string_to_id:
        ret.append(subtoken)
        start = end
        break

    else:  # Did not break
      # If there is no possible encoding of the escaped token then one of the
      # characters in the token is not in the alphabet. This should be
      # impossible and would be indicative of a bug.
      assert False, "Token substring not found in subtoken vocabulary."

  return ret

1 19 192 1929 1929_出现72次
9 92 929 929_出现72次
2 29 29_出现72次
9出现72次

for token, count in six.iteritems(token_counts):
  iter_start_time = time.time()
  escaped_token = _escape_token(token, self._alphabet)
  subtokens = self._escaped_token_to_subtoken_strings(escaped_token)
  start = 0
  for subtoken in subtokens:
    last_position = len(escaped_token) + 1
    if max_subtoken_length is not None:
      last_position = min(last_position, start + max_subtoken_length)

    for end in range(start + 1, last_position):
      new_subtoken = escaped_token[start:end]
      subtoken_counts[new_subtoken] += count
    start += len(subtoken)
  iter_time_secs = time.time() - iter_start_time
  if iter_time_secs > 0.1:
    tf.logging.info(u"Processing token [{0}] took {1} seconds, consider "
                    "setting Text2TextProblem.max_subtoken_length to a "
                    "smaller value.".format(token, iter_time_secs))

# Array of sets of candidate subtoken strings, by length.
len_to_subtoken_strings = []
for subtoken_string, count in six.iteritems(subtoken_counts):
  lsub = len(subtoken_string)
  if count >= min_count:
    while len(len_to_subtoken_strings) <= lsub:
      len_to_subtoken_strings.append(set())
    len_to_subtoken_strings[lsub].add(subtoken_string)

# Consider the candidates longest to shortest, so that if we accept
# a longer subtoken string, we can decrement the counts of its prefixes.
new_subtoken_strings = []
for lsub in range(len(len_to_subtoken_strings) - 1, 0, -1):
  subtoken_strings = len_to_subtoken_strings[lsub]
  for subtoken_string in subtoken_strings:
    count = subtoken_counts[subtoken_string]
    if count >= min_count:
      # Exclude alphabet tokens here, as they must be included later,
      # explicitly, regardless of count.
      if subtoken_string not in self._alphabet:
        new_subtoken_strings.append((count, subtoken_string))
      for l in range(1, lsub):
        subtoken_counts[subtoken_string[:l]] -= count

# Include the alphabet explicitly to guarantee all strings are encodable.
new_subtoken_strings.extend((subtoken_counts.get(a, 0), a)
                            for a in self._alphabet)
new_subtoken_strings.sort(reverse=True)

# Reinitialize to the candidate vocabulary.
new_subtoken_strings = [subtoken for _, subtoken in new_subtoken_strings]
if reserved_tokens:
  escaped_reserved_tokens = [
      _escape_token(native_to_unicode(t), self._alphabet)
      for t in reserved_tokens
  ]
  new_subtoken_strings = escaped_reserved_tokens + new_subtoken_strings

self._init_subtokens_from_list(new_subtoken_strings)
tf.logging.info("vocab_size = %d" % self.vocab_size)

d de dee deep deepe deepen deepens deepens_
e ee eep eepe eepen eepens eepens_
e ep epe epen epens epens_
...
n ns ns_
s s_

d de dee deep deepe deepen deepens deepens_
n ns ns_