from requests_html import HTMLSession import pydantic class WineModel(pydantic.BaseModel): name: str price: str varietal: str description: str session = HTMLSession() import random URL = "https://www.wines.com/wine-varietals/" response = session.get(URL) varietals = [varietal.text for varietal in response.html.find("h5")] URL = "https://top100.winespectator.com/lists/" response = session.get(URL) for row in response.html.find("tbody", first=True).find("tr"): wine_details = row.find(".wineName", first=True) wine_name = wine_details.find("span.sort-text", first=True).text *location, year, extra = wine_details.text.split(" ") price = row.find(".price", first=True).text.replace("$", "") note = row.find(".tabel-note", first=True).text.split("\n") note.pop() details = "\n".join(note) if year.isdigit() is False: continue if price.isdigit() is False: continue location = " ".join(location).replace(wine_name, "").strip() price = "$" + price varietal = random.choice(varietals) wine = WineModel( name=wine_name, price=price, varietal=varietal, description=details ) # print(wine.json()) URL = "https://papaplatoon-wine-api.herokuapp.com/api/wines/" response = session.post(URL, data=wine.json()) print(response.status_code) print(response.json()) # print(wine_name, location, year, price) # print(varietal) # print(details) # print() #