init directml chat server

Author: tjtanaa

.env.example

@@ -0,0 +1,5 @@
+PORT=6979
+HOST="0.0.0.0"
+UVICORN_LOG_LEVEL="debug"
+MODEL_PATH=''
+SERVED_MODEL_NAME='phi3-mini-int4'

.gitignore

@@ -0,0 +1,10 @@
+# Environment
+.env 
+phi3*.py
+test_phi3*
+
+# Python
+**/__pycache__
+**.egg-info
+
+scripts/*.ps1

pyproject.toml

@@ -0,0 +1,125 @@
+# See https://gitlab.liris.cnrs.fr/pagoda/tools/mkdocs_template/-/blob/master/user_config/pyproject.toml
+
+# -----------------------------------------------------------------------------
+# Pytest configuration
+# https://docs.pytest.org/en/latest/customize.html?highlight=pyproject#pyproject-toml
+
+[tool.pytest.ini_options]
+log_cli = true
+asyncio_mode = "auto"
+# log_cli_level = "DEBUG"
+addopts = "--cov=embeddedllm --doctest-modules"
+testpaths = ["tests"]
+filterwarnings = [
+ "ignore::DeprecationWarning:tensorflow.*",
+ "ignore::DeprecationWarning:tensorboard.*",
+ "ignore::DeprecationWarning:matplotlib.*",
+ "ignore::DeprecationWarning:flatbuffers.*",
+]
+
+
+# -----------------------------------------------------------------------------
+# Black (Option-less formatter) configuration
+# https://black.readthedocs.io/en/stable/index.html
+
+[tool.black]
+line-length = 99
+target-version = ["py310"]
+include = '\.pyi?$|\.ipynb'
+
+# -----------------------------------------------------------------------------
+# For sorting imports
+# This is used by VS Code to sort imports
+# https://code.visualstudio.com/docs/python/editing#_sort-imports
+# https://timothycrosley.github.io/isort/
+
+[tool.isort]
+# Profile
+# Base profile type to use for configuration. Profiles include: black, django,
+# pycharm, google, open_stack, plone, attrs, hug. As well as any shared profiles.
+# Default: ``
+profile = "black"
+# Treat project as a git repository and ignore files listed in .gitignore
+# Default: `False`
+skip_gitignore = true
+# The max length of an import line (used for wrapping long imports).
+# Default: `79`
+line_length = 99
+known_first_party = []
+
+# -----------------------------------------------------------------------------
+# setuptools
+# https://setuptools.pypa.io/en/latest/userguide/pyproject_config.html
+
+[build-system]
+# setuptools-scm considers all files tracked by git to be data files
+requires = ["setuptools>=62.0", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "embeddedllm"
+description = "EmbeddedLLM: API server for Embedded Device Deployment. Currently support ONNX-DirectML."
+readme = "README.md"
+requires-python = "~=3.10"
+# keywords = ["one", "two"]
+license = { text = "Proprietary" }
+classifiers = [ # https://pypi.org/classifiers/
+ "Development Status :: 3 - Alpha",
+ "Programming Language :: Python :: 3 :: Only",
+ "Intended Audience :: Information Technology",
+ "Operating System :: Unix",
+]
+dependencies = [
+ "huggingface-hub[cli]",
+ "fastapi~=0.110.0",
+ "gunicorn~=21.2.0",
+ "loguru~=0.7.2",
+ "numpy~=1.26.4",
+ "pydantic-settings>=2.2.1",
+ "pydantic~=2.6.3",
+ "onnxruntime-directml",
+ "onnxruntime-genai-directml",
+ "loguru",
+ "openai",
+ "torch",
+ "transformers",
+ "uvicorn"
+] # Sort your dependencies https://sortmylist.com/
+dynamic = ["version"]
+
+[project.optional-dependencies]
+lint = ["black~=24.4.2", "flake8~=7.0.0"]
+test = [
+ "flaky~=3.7.0",
+ "locust~=2.24.1",
+ "mypy~=1.5.1",
+ "pytest-cov~=4.1.0",
+ "pytest~=7.4.2",
+]
+docs = [
+ "furo~=2023.9.10", # Sphinx theme (nice looking, with dark mode)
+ "myst-parser~=2.0.0",
+ "sphinx-autobuild~=2021.3.14",
+ "sphinx-copybutton~=0.5.2",
+ "sphinx~=7.2.6",
+ "sphinx_rtd_theme~=1.3.0", # Sphinx theme
+]
+build = [
+ "build",
+ "twine",
+] # https://realpython.com/pypi-publish-python-package/#build-your-package
+all = [
+ "embeddedllm[lint,test,docs,build]", # https://hynek.me/articles/python-recursive-optional-dependencies/
+]
+
+# [project.scripts]
+# embeddedllm = "embeddedllm.scripts.example:main_cli"
+
+[tool.setuptools.dynamic]
+version = { attr = "embeddedllm.version.__version__" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+owl = ["**/*.json"]

scripts/python/get_model.py

@@ -0,0 +1,15 @@
+import httpx
+import asyncio
+
+async def fetch_models():
+ url = "http://localhost:6979/v1/models"
+ async with httpx.AsyncClient() as client:
+ response = await client.get(url)
+ return response.json()
+
+async def main():
+ models = await fetch_models()
+ print(models)
+
+if __name__ == "__main__":
+ asyncio.run(main())

scripts/python/httpx_client.py

@@ -0,0 +1,25 @@
+import httpx
+import asyncio
+
+async def stream_chat_completion(url: str, payload: dict):
+ async with httpx.AsyncClient() as client:
+ async with client.stream("POST", url, json=payload) as response:
+ if response.status_code == 200:
+ async for data in response.aiter_bytes():
+ if data:
+ print(data.decode('utf-8'))
+ else:
+ print(f"Error: {response.status_code}")
+ print(await response.text())
+
+# Example usage
+if __name__ == "__main__":
+ url = "http://localhost:6979/v1/chat/completions"
+ payload = {
+ "messages": [{"role": "user", "content": "Hello!"}],
+ "model": "phi3-mini-int4",
+ "max_tokens": 80,
+ "temperature": 0.0,
+ "stream": True
+ }
+ asyncio.run(stream_chat_completion(url, payload))

scripts/python/litellm_client.py

@@ -0,0 +1,15 @@
+import litellm 
+
+messages = [{"role": "user", "content": "Hey, how's it going?"}]
+
+response = litellm.completion(
+ model="phi3-mini-int4", # pass the vllm model name
+ messages=messages,
+ api_base="http://localhost:6979/v1",
+ api_key="EMPTY",
+ temperature=0,
+ max_tokens=80, stream=True,
+ custom_llm_provider="openai")
+
+for part in response:
+ print(part.choices[0].delta.content or "")

scripts/python/openai_client.py

@@ -0,0 +1,40 @@
+# from openai import AsyncOpenAI
+# import asyncio
+# import time
+
+# client = AsyncOpenAI(
+# base_url="http://localhost:6979/v1",
+# api_key='ellm'
+# )
+
+
+# async def main():
+# stream = await client.chat.completions.create(
+# model="phi3-mini-int4",
+# messages=[{"role": "user", "content": "Say this is a test"}],
+# max_tokens=80,
+# temperature=0,
+# stream=True,
+# )
+# print(stream)
+# async for chunk in stream:
+# print(chunk.choices[0].delta.content or "", end="", flush=True)
+
+
+# asyncio.run(main())
+
+from openai import OpenAI
+
+client = OpenAI(
+ base_url="http://localhost:6979/v1",
+ api_key='ellm')
+
+stream = client.chat.completions.create(
+ model="phi3-mini-int4",
+ messages=[{"role": "user", "content": "Say this is a test"}],
+ max_tokens=80,
+ temperature=0,
+ stream=True,
+)
+for chunk in stream:
+ print(chunk.choices[0].delta.content or "", end="")