#!/usr/bin/env python3
import sys
sys.path.insert(0, 'c:/MONIKA')
from salience_os_seed.proto_lm.vocab import Vocabulary

v = Vocabulary()
v.tokens.append(" I")
v.tokens.append(" am")
v._refresh_index()

text = "Hello I am test"
print(f"Text: '{text}'")
print(f"Vocab has ' I': {' I' in v.token_to_id}")
print(f"Vocab has ' am': {' am' in v.token_to_id}")

# Test encode
symbols = v.encode(text)
print(f"Encoded symbols: {symbols}")

# Test encode_ids_readonly
ids = v.encode_ids_readonly(text)
print(f"Token IDs: {ids}")
print(f"Decoded: '{v.decode_ids(ids)}'")