{
  "model": {
    "vocab_size": 193,
    "embedding_dim": 512,
    "num_layers": 6,
    "num_heads": 8,
    "feedforward_dim": null,
    "max_seq_length": 512,
    "dropout": 0.1,
    "use_memory_buffer": true,
    "memory_buffer_size": 32,
    "use_formula_attention": true,
    "formula_w1": 0.4,
    "formula_w2": 0.3,
    "formula_w3": 0.3,
    "formula_lambda": 0.1,
    "formula_k": 0.2,
    "formula_learnable_weights": true,
    "formula_learnable_decay": true
  },
  "training": {
    "batch_size": 2,
    "learning_rate": 0.0001,
    "weight_decay": 0.01,
    "num_epochs": 1,
    "warmup_steps": 1000,
    "max_grad_norm": 1.0,
    "log_interval": 100,
    "save_dir": "./checkpoints/test_run",
    "device": "cuda",
    "train_data_path": "./data/sample_train.txt",
    "val_data_path": "./data/sample_valid.txt",
    "max_new_tokens": 50,
    "temperature": 1.0,
    "top_k": null,
    "top_p": 1.0
  }
}