Spaces:
Running
Running
Replace ⎵ by space in the dataframe otherwise it will not search correctly
Browse files
app.py
CHANGED
|
@@ -12,7 +12,7 @@ text3 = solara.reactive("")
|
|
| 12 |
# Create dataframe mapping token IDs and tokens
|
| 13 |
df = pd.DataFrame()
|
| 14 |
df["token ID"] = range(tokenizer.vocab_size)
|
| 15 |
-
df["token"] = [
|
| 16 |
|
| 17 |
@solara.component
|
| 18 |
def Page():
|
|
|
|
| 12 |
# Create dataframe mapping token IDs and tokens
|
| 13 |
df = pd.DataFrame()
|
| 14 |
df["token ID"] = range(tokenizer.vocab_size)
|
| 15 |
+
df["token"] = [tokenizer.decode([i]) for i in range(tokenizer.vocab_size)]
|
| 16 |
|
| 17 |
@solara.component
|
| 18 |
def Page():
|