63 lines
2.6 KiB
Python
63 lines
2.6 KiB
Python
import os
|
|
import re
|
|
|
|
def clean_and_combine_gutenberg(input_folder, output_filepath):
|
|
# Standard Gutenberg start and end markers (case-insensitive regex)
|
|
start_pattern = re.compile(r"\*\*\*?\s*START OF (THE|THIS) PROJECT GUTENBERG", re.IGNORECASE)
|
|
end_pattern = re.compile(r"\*\*\*?\s*END OF (THE|THIS) PROJECT GUTENBERG", re.IGNORECASE)
|
|
|
|
books_processed = 0
|
|
|
|
with open(output_filepath, "w", encoding="utf-8") as outfile:
|
|
# Loop through all files in the target directory
|
|
for filename in os.listdir(input_folder):
|
|
if filename.endswith(".txt"):
|
|
file_path = os.path.join(input_folder, filename)
|
|
print(file_path)
|
|
|
|
with open(file_path, "r", encoding="utf-8", errors="ignore") as infile:
|
|
lines = infile.readlines()
|
|
|
|
inside_book = False
|
|
cleaned_content = []
|
|
|
|
for line in lines:
|
|
# Check for the end marker first to stop collecting
|
|
if end_pattern.search(line):
|
|
inside_book = False
|
|
break
|
|
|
|
# If we are inside the book boundaries, save the line
|
|
if inside_book:
|
|
cleaned_content.append(line)
|
|
|
|
# Check for the start marker to begin collecting on the NEXT line
|
|
if start_pattern.search(line):
|
|
inside_book = True
|
|
|
|
# If we successfully found content, write it to the master file
|
|
if cleaned_content:
|
|
# Optional: Add a couple of newlines between books so they don't smash together
|
|
outfile.write("".join(cleaned_content))
|
|
outfile.write("\n\n")
|
|
books_processed += 1
|
|
print(True)
|
|
else:
|
|
print(False)
|
|
|
|
print(f"\n All done! Processed {books_processed} books into '{output_filepath}'.")
|
|
|
|
# --- Configuration ---
|
|
# Replace 'my_gutenberg_books' with the path to your folder containing the .txt files
|
|
INPUT_DIRECTORY = "./saved_files"
|
|
OUTPUT_FILE = "combined_training_data.txt"
|
|
|
|
# Run the script
|
|
if __name__ == "__main__":
|
|
# Create a dummy folder if you haven't made one yet
|
|
if not os.path.exists(INPUT_DIRECTORY):
|
|
os.makedirs(INPUT_DIRECTORY)
|
|
print(f"Created folder '{INPUT_DIRECTORY}'. Drop your Project Gutenberg .txt files in there and re-run!")
|
|
else:
|
|
clean_and_combine_gutenberg(INPUT_DIRECTORY, OUTPUT_FILE)
|