Skip to content

Instantly share code, notes, and snippets.

@ShawonAshraf

ShawonAshraf/main.py

Created April 11, 2024 20:11
Show Gist options
  • Save ShawonAshraf/628640f1b39b427e354476f650735d97 to your computer and use it in GitHub Desktop.
Save ShawonAshraf/628640f1b39b427e354476f650735d97 to your computer and use it in GitHub Desktop.
Download ZIP
vllm request
Raw
main.py
import openai
from pydantic import BaseModel, Field
import json
class ResponseStructure(BaseModel):
bookname: str = Field(..., title="Book Name")
author: str = Field(..., title="Author")
reading_list_name: str = Field(..., title="Reading List Name")
client = openai.Client(
api_key="ollama",
base_url="http://gandalf:8000/v1"
)
messages = [
{"role": "system", "content": "You are a helpful assistant designed to output JSON."},
{"role": "user", "content": "Add the book '1984' by 'George Orwell' to the 'Currently Reading' list."},
]
chat_response = client.chat.completions.create(
model="solidrust/Hermes-2-Pro-Mistral-7B-AWQ",
messages=messages,# type: ignore
response_format={"type": "json_object", "schema": ResponseStructure.model_json_schema()}, # type: ignore
temperature=0.0,
)
# json_obj = json.loads(chat_response.choices[0].message.content) # type: ignore
# print(json_obj)
print(chat_response.choices[0].message.content)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment