There was an error while loading. Please reload this page.
1 parent 392a438 commit 1f8b704Copy full SHA for 1f8b704
README.md
@@ -52,7 +52,8 @@ model = AutoModelForCausalLM.from_pretrained(checkpoint).to(device)
52
53
inputs = tokenizer.encode("def print_hello_world():", return_tensors="pt").to(device)
54
outputs = model.generate(inputs)
55
-print(tokenizer.decode(outputs[0]))
+# clean_up_tokenization_spaces=False prevents a tokenizer edge case which can result in spaces being removed around punctuation
56
+print(tokenizer.decode(outputs[0], clean_up_tokenization_spaces=False))
57
```
58
or
59
```python
0 commit comments