update dependencies, add script to create RAG of a github repo, add API endpoint to query it, add chatbot page to interact with it

scripts/finetune.js

@@ -0,0 +1,35 @@
+// scripts/finetune.js
+import { GithubRepoLoader } from '@langchain/community/document_loaders/web/github';
+import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
+import { OpenAIEmbeddings } from '@langchain/openai';
+import { FaissStore } from '@langchain/community/vectorstores/faiss';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+async function finetune() {
+	console.log('Fine-tuning model...');
+
+	const loader = new GithubRepoLoader('https://github.com/silentsilas/Authenticator', {
+		branch: 'develop',
+		recursive: true,
+		unknown: 'warn',
+		maxConcurrency: 5
+	});
+	const docs = await loader.load();
+
+	const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
+	const splitDocs = await textSplitter.splitDocuments(docs);
+
+	const embeddings = new OpenAIEmbeddings();
+	const vectorStore = await FaissStore.fromDocuments(splitDocs, embeddings);
+
+	const directory = join(__dirname, '..', 'vectorstore');
+	await vectorStore.save(directory);
+
+	console.log('Fine-tuning complete. Vector store saved.');
+}
+
+finetune();