training different models

train_v2.py

@@ -84,24 +84,36 @@ def load_or_create_vocab(text_file, vocab_file):
     return vocab
 
 def produce_sample(model, vectorize_layer, vocab, seed, num_generate=300, temperature=0.7):
-    input_chars = vectorize_layer(tf.constant([seed]))
-    input_ids = input_chars[0][:len(seed)].numpy().tolist()
+    # 1. Safely tokenize the seed character-by-character using the vectorizer
+    # We split the string into individual characters so the vectorizer processes them properly
+    seed_chars = tf.strings.unicode_split(seed, input_encoding='UTF-8')
+    input_ids = vectorize_layer(seed_chars).numpy().tolist()
 
     generated_ids = []
     states = None
 
+    # 2. Process the entire seed sequence first to warm up the LSTM states
     predictions, states = model(tf.expand_dims(input_ids, 0), states=states, return_state=True, training=False)
+
+    # Grab the logits for the very last character of the seed
     predictions = predictions[0, -1, :] / temperature
     predicted_id = tf.random.categorical(tf.expand_dims(predictions, 0), num_samples=1)[0, 0].numpy()
     generated_ids.append(predicted_id)
 
+    # 3. Autoregressive loop for the remaining characters
     for _ in range(num_generate - 1):
+        # We pass only the single, last-predicted character ID
         predictions, states = model(tf.expand_dims([predicted_id], 0), states=states, return_state=True, training=False)
         predictions = predictions[0, -1, :] / temperature
         predicted_id = tf.random.categorical(tf.expand_dims(predictions, 0), num_samples=1)[0, 0].numpy()
         generated_ids.append(predicted_id)
 
-    return "".join([vocab[idx] for idx in generated_ids])
+    # 4. Join the vocabulary tokens, ignoring any special [PAD] or [UNK] tokens if they sneak in
+    generated_text = "".join([vocab[idx] for idx in generated_ids if idx < len(vocab)])
+
+    # Return the seed + the newly generated text
+    return seed + generated_text
+
 
 class GenerationCallback(tf.keras.callbacks.Callback):
     def __init__(self, vectorize_layer, vocab, seed_text):
@@ -134,16 +146,16 @@ def main():
     args = parser.parse_args()
 
     # Default settings for clean, scratch runs
-    DEFAULT_EMBEDDING_DIM = 512
-    DEFAULT_RNN_UNITS = 512
-    DEFAULT_NUM_LAYERS = 1
-    DEFAULT_DROPOUT_RATE = 0.3
+    DEFAULT_EMBEDDING_DIM = 256
+    DEFAULT_RNN_UNITS = 1024
+    DEFAULT_NUM_LAYERS = 3
+    DEFAULT_DROPOUT_RATE = 0.4
 
     TEXT_FILE = "emily_post.txt"
     VOCAB_FILE = "vocab.txt"
     SEQ_LENGTH = 512
     BATCH_SIZE = 128
-    EPOCHS_PER_RUN = 200  # Easily run in chunks of 20 epochs
+    EPOCHS_PER_RUN = 20
     BUFFER_SIZE = 10000
     SEED_TEXT = "In the case of unlawful enterance"