Added bpe tokenizer.

2026-04-29 00:17:34 +02:00
parent d1bc5ef8aa
commit 7a67050b77
8 changed files with 401 additions and 2 deletions
--- a/pyproject.toml
+++ b/pyproject.toml
@ -1,7 +1,9 @@
 [project]
 name = "llmfs"
 version = "0.1.0"
-description = "Add your description here"
+description = "LLM from scratch"
 readme = "README.md"
 requires-python = ">=3.13"
-dependencies = []
+dependencies = [
+    "tiktoken>=0.12.0",
+]