async function processAdoc(file, path) {
  console.log(`Processing ${path}`);

  const frontMatterRegex = /^---[\s\S]+?---\n*/;


  const namespace = path.includes('articles/react') ? 'react' : path.includes('articles/lit') ? 'lit' : '';
  if (!namespace) return;

  // Remove front matter. The JS version of asciidoctor doesn't support removing it.
  const noFrontMatter = file.replace(frontMatterRegex, '');

  // Run through asciidoctor to get includes
  const html = asciidoctor.convert(noFrontMatter, {
    attributes: {
      root: process.env.DOCS_ROOT,
      articles: process.env.DOCS_ARTICLES,
      react: namespace === 'react',
      lit: namespace === 'lit'
    },
    safe: 'unsafe',
    base_dir: process.env.DOCS_ARTICLES
  });

  // Extract sections
  const dom = new JSDOM(html);
  const sections = dom.window.document.querySelectorAll('.sect1');

  // Convert section html to plain text to save on tokens
  const plainText = Array.from(sections).map(section => convert(section.innerHTML));

  // Split section content further if needed, filter out short blocks
  const docs = await splitter.createDocuments(plainText);
  const blocks = docs.map(doc => doc.pageContent)
    .filter(block => block.length > 200);

  await createAndSaveEmbeddings(blocks, path, namespace);
}

async function createAndSaveEmbeddings(blocks, path, namespace) {

  // OpenAI suggests removing newlines for better performance when creating embeddings. 
  // Don't remove them from the source.
  const withoutNewlines = blocks.map(block => block.replace(/\n/g, ' '));
  const embeddings = await getEmbeddings(withoutNewlines);
  const vectors = embeddings.map((embedding, i) => ({
    id: nanoid(),
    values: embedding,
    metadata: {
      path: path,
      text: blocks[i]
    }
  }));

  await pinecone.upsert({
    upsertRequest: {
      vectors,
      namespace
    }
  });
}

export async function getEmbeddings(texts) {
  const response = await openai.createEmbedding({
    model: 'text-embedding-ada-002',
    input: texts
  });
  return response.data.data.map((item) => item.embedding);
}

async function getMessagesWithContext(messages: ChatCompletionRequestMessage[], frontend: string) {

  // Ensure that there are only messages from the user and assistant, trim input
  const historyMessages = sanitizeMessages(messages);

  // Send all messages to OpenAI for moderation.
  // Throws exception if flagged -> should be handled properly in a real app.
  await moderate(historyMessages);

  // Extract the last user message to get the question
  const [userMessage] = historyMessages.filter(({role}) => role === ChatCompletionRequestMessageRoleEnum.User).slice(-1)

  // Create an embedding for the user's question
  const embedding = await createEmbedding(userMessage.content);

  // Find the most similar documents to the user's question
  const docSections = await findSimilarDocuments(embedding, 10, frontend);

  // Get at most 1536 tokens of documentation as context
  const contextString = await getContextString(docSections, 1536);

  // The messages that set up the context for the question
  const initMessages: ChatCompletionRequestMessage[] = [
    {
      role: ChatCompletionRequestMessageRoleEnum.System,
      content: codeBlock`
          ${oneLine`
            You are Hilla AI. You love to help developers! 
            Answer the user's question given the following
            information from the Hilla documentation.
          `}
        `
    },
    {
      role: ChatCompletionRequestMessageRoleEnum.User,
      content: codeBlock`
          Here is the Hilla documentation:
          """
          ${contextString}
          """
        `
    },
    {
      role: ChatCompletionRequestMessageRoleEnum.User,
      content: codeBlock`
          ${oneLine`
            Answer all future questions using only the above        
            documentation and your knowledge of the 
            ${frontend === 'react' ? 'React' : 'Lit'} library
          `}
          ${oneLine`
            You must also follow the below rules when answering:
          `}
          ${oneLine`
            - Do not make up answers that are not provided 
              in the documentation 
          `}
          ${oneLine`
            - If you are unsure and the answer is not explicitly 
              written in the documentation context, say 
              "Sorry, I don't know how to help with that"
          `}
          ${oneLine`
            - Prefer splitting your response into 
              multiple paragraphs
          `}
          ${oneLine`
            - Output as markdown
          `}
          ${oneLine`
            - Always include code snippets if available
          `}
        `
    }
  ];

  // Cap the messages to fit the max token count, removing earlier messages if necessary
  return capMessages(
    initMessages,
    historyMessages
  );
}

export default async function handler(req: NextRequest) {
  // All the non-system messages up until now along with 
  // the framework we should use for the context.
  const {messages, frontend} = (await req.json()) as {
    messages: ChatCompletionRequestMessage[],
    frontend: string
  };
  const completionMessages = await getMessagesWithContext(messages, frontend);
  const stream = await streamChatCompletion(completionMessages, MAX_RESPONSE_TOKENS);
  return new Response(stream);
}