#!/usr/bin/env python3
import sys
sys.path.insert(0, 'c:/MONIKA')
from salience_os_seed.proto_lm.vocab import Vocabulary

v = Vocabulary()
print(f"Initial merges: {len(v.merges)}")
print(f"Initial tokens: {v.size()}")

# Encode a string
text = "Hello"
result = v.encode(text)
print(f"\nEncoding '{text}':")
print(f"Result: {result}")
print(f"After encoding - merges: {len(v.merges)}")

# Try encode_ids_readonly
ids1 = v.encode_ids("Hello")
print(f"\nencode_ids: {ids1}")
print(f"Tokens: {[v.tokens[i] for i in ids1]}")

v2 = Vocabulary()
ids2 = v2.encode_ids_readonly("Hello")
print(f"\nencode_ids_readonly (fresh vocab): {ids2}")
print(f"Tokens: {[v2.tokens[i] for i in ids2]}")