"""
Create a larger sample dataset for training and evaluation.
"""

import os
import random


def create_larger_sample_data(output_dir='./data'):
    """Create a larger sample training dataset."""
    os.makedirs(output_dir, exist_ok=True)
    
    # Generate a larger corpus with repeated patterns and variations
    base_texts = [
        "The novel AI model uses a groundbreaking formula for information processing.",
        "Unlike traditional transformers, this model scores information based on multiple factors.",
        "The scoring formula combines novelty, retention, and payoff to determine importance.",
        "Novelty measures how much new information a token provides relative to context.",
        "Retention estimates the long-term value and memorability of information.",
        "Payoff computes the immediate utility and relevance of the current token.",
        "Continuity ensures that selected information maintains coherence with context.",
        "Fatigue penalizes redundant information that has appeared recently.",
        "Time decay applies an exponential decay function based on sequence position.",
        "The formula allows the model to dynamically prioritize information during processing.",
        "This approach differs fundamentally from standard attention mechanisms.",
        "Traditional attention uses dot-product similarity between queries and keys.",
        "Our formula-based approach considers multiple dimensions of information quality.",
        "The weights for novelty, retention, and payoff are learnable parameters.",
        "The model adapts these weights during training to optimize performance.",
        "Experimental results show promising improvements in information selection.",
        "The architecture maintains compatibility with existing transformer infrastructure.",
        "Training can be performed using standard optimization techniques.",
        "Gradient descent works well with the differentiable formula components.",
        "The memory buffer tracks recent embeddings for fatigue computation.",
        "Each component of the formula is computed using small neural networks.",
        "The novelty network compares current and context embeddings effectively.",
        "The retention network evaluates future importance accurately.",
        "The payoff network measures immediate relevance precisely.",
        "The continuity network ensures semantic coherence throughout the sequence.",
        "The fatigue network compares against recent items stored in memory.",
        "All components are differentiable and enable end-to-end training.",
        "The model supports standard language modeling tasks efficiently.",
        "Text generation uses the formula to guide token selection intelligently.",
        "The architecture scales well with increased model size.",
        "Larger models show improved performance on various benchmarks.",
        "Evaluation metrics include perplexity and accuracy measurements.",
        "The model can be fine-tuned for specific domains successfully.",
        "Transfer learning works effectively with this novel architecture.",
    ]
    
    # Generate more variations and combinations
    all_texts = []
    
    # Add base texts multiple times with variations
    for _ in range(50):
        for base in base_texts:
            # Add some variations
            variations = [
                base,
                base.replace("model", "system"),
                base.replace("formula", "equation"),
                base.replace("information", "data"),
            ]
            all_texts.extend(variations)
    
    # Shuffle
    random.seed(42)
    random.shuffle(all_texts)
    
    # Split into train/valid/test
    total = len(all_texts)
    train_size = int(total * 0.8)
    val_size = int(total * 0.1)
    
    # Write training data
    with open(os.path.join(output_dir, 'sample_train.txt'), 'w', encoding='utf-8') as f:
        f.write('\n\n'.join(all_texts[:train_size]))
    
    # Write validation data
    with open(os.path.join(output_dir, 'sample_valid.txt'), 'w', encoding='utf-8') as f:
        f.write('\n\n'.join(all_texts[train_size:train_size+val_size]))
    
    # Write test data
    with open(os.path.join(output_dir, 'sample_test.txt'), 'w', encoding='utf-8') as f:
        f.write('\n\n'.join(all_texts[train_size+val_size:]))
    
    print(f"Created larger sample data in {output_dir}/")
    print(f"  - sample_train.txt ({train_size:,} texts)")
    print(f"  - sample_valid.txt ({val_size:,} texts)")
    print(f"  - sample_test.txt ({total - train_size - val_size:,} texts)")


if __name__ == '__main__':
    create_larger_sample_data()