// Create a smol embedding; CloudflareWorkersAIEmbeddings is provided by langchain
function setupEmbeddings(env: Env): CloudflareWorkersAIEmbeddings {
  return new CloudflareWorkersAIEmbeddings({
    binding: env.AI as unknown as Fetcher,
    modelName: '@cf/baai/bge-small-en-v1.5',
    stripNewLines: true,
    onFailedAttempt: (error) => {
      console.log('Failed attempt on embed', error);
    }
  });
};

// CloudflareVectorizeStore is provided by langchain
function vectorStorage(env: Env): CloudflareVectorizeStore {
  return new CloudflareVectorizeStore(setupEmbeddings(env), {
    index: env.V1_VECTORIZE, // index
    onFailedAttempt: (error) => {
      console.log('Failed attempt on vectorize', error);
    },
  });
};

// Just a wiki article, tried to find smth small
const content = 'https://en.wikipedia.org/wiki/Attiki,_Athens';

// Get the Cheerio Web Loader and split docs
const rawDocs = await new CheerioWebBaseLoader(content as string).load(); // tried loadAndSplit() too

// HTML -> Docs splitter
const splitter = RecursiveCharacterTextSplitter.fromLanguage('html');
const sequence = splitter.pipe(new HtmlToTextTransformer());

// Make the sequence pass through HtmlToTextTransformer() to get the final docs
const documents = await sequence.invoke(rawDocs);

// This call triggers the error.
await vectorStorage(env).addDocuments(documents);