Update README.md
Browse filesTrailing space breaks tokenization
README.md
CHANGED
|
@@ -75,7 +75,7 @@ Proceed as usual with HuggingFace:
|
|
| 75 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
olmo = AutoModelForCausalLM.from_pretrained("allenai/OLMo-7B-0424-hf")
|
| 77 |
tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-7B-0424-hf")
|
| 78 |
-
message = ["Language modeling is
|
| 79 |
inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
|
| 80 |
# optional verifying cuda
|
| 81 |
# inputs = {k: v.to('cuda') for k,v in inputs.items()}
|
|
|
|
| 75 |
from transformers import AutoModelForCausalLM, AutoTokenizer
|
| 76 |
olmo = AutoModelForCausalLM.from_pretrained("allenai/OLMo-7B-0424-hf")
|
| 77 |
tokenizer = AutoTokenizer.from_pretrained("allenai/OLMo-7B-0424-hf")
|
| 78 |
+
message = ["Language modeling is"]
|
| 79 |
inputs = tokenizer(message, return_tensors='pt', return_token_type_ids=False)
|
| 80 |
# optional verifying cuda
|
| 81 |
# inputs = {k: v.to('cuda') for k,v in inputs.items()}
|