import numpy as np from transformers import AutoTokenizer from openvino import compile_model, save_model from openvino_tokenizers import convert_tokenizer hf_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased") ov_tokenizer = convert_tokenizer(hf_tokenizer) compiled_tokenzier = compile_model(ov_tokenizer) text_input = ["Test string"] hf_output = hf_tokenizer(text_input, return_tensors="np") ov_output = compiled_tokenzier(text_input) for output_name in hf_output: print(f"OpenVINO {output_name} = {ov_output[output_name]}") print(f"HuggingFace {output_name} = {hf_output[output_name]}") # OpenVINO input_ids = [[ 101 3231 5164 102]] # HuggingFace input_ids = [[ 101 3231 5164 102]] # OpenVINO token_type_ids = [[0 0 0 0]] # HuggingFace token_type_ids = [[0 0 0 0]] # OpenVINO attention_mask = [[1 1 1 1]] # HuggingFace attention_mask = [[1 1 1 1]] # save tokenizer for later use save_model(ov_tokenizer, "openvino_tokenizer.xml") loaded_tokenizer = compile_model("openvino_tokenizer.xml") loaded_ov_output = loaded_tokenizer(text_input) for output_name in hf_output: assert np.all(loaded_ov_output[output_name] == ov_output[output_name])

from transformers import AutoTokenizer, AutoModelForSequenceClassification from openvino import compile_model, convert_model from openvino_tokenizers import convert_tokenizer, connect_models checkpoint = "mrm8488/bert-tiny-finetuned-sms-spam-detection" hf_tokenizer = AutoTokenizer.from_pretrained(checkpoint) hf_model = AutoModelForSequenceClassification.from_pretrained(checkpoint) text_input = ["Free money!!!"] hf_input = hf_tokenizer(text_input, return_tensors="pt") hf_output = hf_model(**hf_input) ov_tokenizer = convert_tokenizer(hf_tokenizer) ov_model = convert_model(hf_model, example_input=hf_input.data) combined_model = connect_models(ov_tokenizer, ov_model) compiled_combined_model = compile_model(combined_model) openvino_output = compiled_combined_model(text_input) print(f"OpenVINO logits: {openvino_output['logits']}") # OpenVINO logits: [[ 1.2007061 -1.4698029]] print(f"HuggingFace logits {hf_output.logits}") # HuggingFace logits tensor([[ 1.2007, -1.4698]], grad_fn=<AddmmBackward0>)

import numpy as np import openvino_tokenizers from openvino import Core core = Core() # detokenizer from codellama sentencepiece model compiled_detokenizer = core.compile_model("detokenizer.xml") token_ids = np.random.randint(100, 1000, size=(3, 5)) openvino_output = compiled_detokenizer(token_ids) print(openvino_output["string_output"]) # ['sc�ouition�', 'intvenord hasient', 'g shouldwer M more']

import numpy as np from openvino import compile_model, convert_model from openvino_tokenizers import add_greedy_decoding, convert_tokenizer from transformers import AutoModelForCausalLM, AutoTokenizer model_checkpoint = "JackFram/llama-68m" hf_tokenizer = AutoTokenizer.from_pretrained(model_checkpoint) hf_model = AutoModelForCausalLM.from_pretrained(model_checkpoint, use_cache=False) # convert hf tokenizer text_input = ["Quick brown fox jumped "] ov_tokenizer, ov_detokenizer = convert_tokenizer(hf_tokenizer, with_detokenizer=True) compiled_tokenizer = compile_model(ov_tokenizer) # transform input text into tokens ov_input = compiled_tokenizer(text_input) hf_input = hf_tokenizer(text_input, return_tensors="pt") # convert Pytorch model to OpenVINO IR and add greedy decoding pipeline to it ov_model = convert_model(hf_model, example_input=hf_input.data) ov_model_with_greedy_decoding = add_greedy_decoding(ov_model) compiled_model = compile_model(ov_model_with_greedy_decoding) # generate new tokens new_tokens_size = 10 prompt_size = ov_input["input_ids"].shape[-1] input_dict = { output.any_name: np.hstack([tensor, np.zeros(shape=(1, new_tokens_size), dtype=np.int_)]) for output, tensor in ov_input.items() } for idx in range(prompt_size, prompt_size + new_tokens_size): output = compiled_model(input_dict)["token_ids"] input_dict["input_ids"][:, idx] = output[:, idx - 1] input_dict["attention_mask"][:, idx] = 1 ov_token_ids = input_dict["input_ids"] hf_token_ids = hf_model.generate( **hf_input, min_new_tokens=new_tokens_size, max_new_tokens=new_tokens_size, temperature=0, # greedy decoding ) # decode model output compiled_detokenizer = compile_model(ov_detokenizer) ov_output = compiled_detokenizer(ov_token_ids)["string_output"] hf_output = hf_tokenizer.batch_decode(hf_token_ids, skip_special_tokens=True) print(f"OpenVINO output string: `{ov_output}`") # OpenVINO output string: `['Quick brown fox was walking through the forest. He was looking for something']` print(f"HuggingFace output string: `{hf_output}`") # HuggingFace output string: `['Quick brown fox was walking through the forest. He was looking for something']`

import numpy as np import tensorflow_hub as hub import tensorflow_text # register tf text ops from openvino import convert_model, compile_model import openvino_tokenizers # register ov tokenizer ops and translators sentences = ["dog", "I cuccioli sono carini.", "私は犬と一緒にビーチを散歩するのが好きです"] tf_embed = hub.load( "https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/" "TensorFlow2/variations/multilingual/versions/2" ) # convert model that uses Sentencepiece tokenizer op from TF Text ov_model = convert_model(tf_embed) ov_embed = compile_model(ov_model, "CPU") ov_result = ov_embed(sentences)[ov_embed.output()] tf_result = tf_embed(sentences) assert np.all(np.isclose(ov_result, tf_result, atol=1e-4))

from urllib.request import urlopen from openvino import compile_model from openvino_tokenizers import build_rwkv_tokenizer rwkv_vocab_url = ( "https://raw.githubusercontent.com/BlinkDL/ChatRWKV/main/tokenizer/rwkv_vocab_v20230424.txt" ) with urlopen(rwkv_vocab_url) as vocab_file: vocab = map(bytes.decode, vocab_file) tokenizer, detokenizer = build_rwkv_tokenizer(vocab) tokenizer, detokenizer = compile_model(tokenizer), compile_model(detokenizer) print(tokenized := tokenizer(["Test string"])["input_ids"]) # [[24235 47429]] print(detokenizer(tokenized)["string_output"]) # ['Test string']

from transformers import AutoTokenizer import openvino as ov from openvino_tokenizers import convert_tokenizer model_id = "unsloth/DeepSeek-R1-Distill-Qwen-1.5B-GGUF" filename = "DeepSeek-R1-Distill-Qwen-1.5B-Q2_K.gguf" hf_tokenizer = AutoTokenizer.from_pretrained(model_id, gguf_file=filename) ov_tokenizer, ov_detokenizer = convert_tokenizer(hf_tokenizer, with_detokenizer=True) ov_tokenizer, ov_detokenizer = ov.compile_model(ov_tokenizer), ov.compile_model(ov_detokenizer) print(ov_res := ov_tokenizer(["Test string"])["input_ids"]) # [[2271 914]] print(ov_detokenizer(ov_res)["string_output"]) # ['Test string']