From deb6052de8cb5208b1e206fbb577c387f4e8d736 Mon Sep 17 00:00:00 2001 From: Silas Date: Sun, 9 Apr 2023 15:33:46 -0400 Subject: [PATCH] get it working with text files :tada: --- package-lock.json | 573 ++++++++++++++++++++++++++++++- package.json | 7 +- src/lib/env.ts | 5 + src/lib/shared/stores/prompt.ts | 5 - src/lib/shared/stores/summary.ts | 5 + src/lib/types/env.d.ts | 3 + src/routes/+page.svelte | 8 +- src/routes/+server.ts | 44 +-- src/routes/about/+page.svelte | 22 +- src/routes/chat/+page.svelte | 10 +- 10 files changed, 626 insertions(+), 56 deletions(-) create mode 100644 src/lib/env.ts delete mode 100644 src/lib/shared/stores/prompt.ts create mode 100644 src/lib/shared/stores/summary.ts create mode 100644 src/lib/types/env.d.ts diff --git a/package-lock.json b/package-lock.json index 69e4b24..00e99d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,10 @@ "": { "name": "sumi", "version": "0.0.1", + "dependencies": { + "@dqbd/tiktoken": "^1.0.4", + "langchain": "^0.0.51" + }, "devDependencies": { "@playwright/test": "^1.28.1", "@skeletonlabs/skeleton": "^1.1.0", @@ -18,6 +22,7 @@ "@typescript-eslint/eslint-plugin": "^5.45.0", "@typescript-eslint/parser": "^5.45.0", "autoprefixer": "^10.4.14", + "dotenv": "^16.0.3", "eslint": "^8.28.0", "eslint-config-prettier": "^8.5.0", "eslint-plugin-svelte3": "^4.0.0", @@ -33,6 +38,20 @@ "vitest": "^0.25.3" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.4.3.tgz", + "integrity": "sha512-SZrlXvjUUYT9rPmSzlTtmVk1OjVNpkCzILRluhiYwNcxXfQyvPJDi0CI6PyymygcgtqEF5EVqhKmC/PtPsNEIw==", + "dependencies": { + "@fortaine/fetch-event-source": "^3.0.6", + "cross-fetch": "^3.1.5" + } + }, + "node_modules/@dqbd/tiktoken": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.4.tgz", + "integrity": "sha512-C0HrJj2RNlsB3wslfNHGNH8xN7QQMki+y4JkUor/GE+oIfPvH7yVep9l1/2powam8AAH6+gdv5MggA5gsszweg==" + }, "node_modules/@esbuild/android-arm": { "version": "0.17.15", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.15.tgz", @@ -441,6 +460,14 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@fortaine/fetch-event-source": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@fortaine/fetch-event-source/-/fetch-event-source-3.0.6.tgz", + "integrity": "sha512-621GAuLMvKtyZQ3IA6nlDWhV1V/7PGOTNIGLUifxt0KzM+dZIweJ6F3XvQF3QnqeNfS1N7WQ0Kil1Di/lhChEw==", + "engines": { + "node": ">=16.15" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -1530,6 +1557,11 @@ "integrity": "sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==", "dev": true }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "node_modules/autoprefixer": { "version": "10.4.14", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz", @@ -1563,6 +1595,14 @@ "postcss": "^8.1.0" } }, + "node_modules/axios": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "dependencies": { + "follow-redirects": "^1.14.8" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -1573,7 +1613,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true, "engines": { "node": ">=8" } @@ -1609,6 +1648,11 @@ "node": ">=8" } }, + "node_modules/browser-or-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", + "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" + }, "node_modules/browserslist": { "version": "4.21.5", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", @@ -1814,6 +1858,17 @@ "color-support": "bin.js" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -1844,6 +1899,33 @@ "node": ">= 0.6" } }, + "node_modules/cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "dependencies": { + "node-fetch": "2.6.7" + } + }, + "node_modules/cross-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -1914,6 +1996,14 @@ "node": ">=0.10.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -1980,6 +2070,15 @@ "node": ">=6.0.0" } }, + "node_modules/dotenv": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", + "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.356", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.356.tgz", @@ -2271,6 +2370,21 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "node_modules/exponential-backoff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", + "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==" + }, + "node_modules/expr-eval": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/expr-eval/-/expr-eval-2.0.2.tgz", + "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -2372,6 +2486,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "bin": { + "flat": "cli.js" + } + }, "node_modules/flat-cache": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", @@ -2391,6 +2513,38 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, + "node_modules/follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fraction.js": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", @@ -2678,7 +2832,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "dependencies": { "binary-extensions": "^2.0.0" }, @@ -2795,6 +2948,14 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/kleur": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", @@ -2804,6 +2965,118 @@ "node": ">=6" } }, + "node_modules/langchain": { + "version": "0.0.51", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.51.tgz", + "integrity": "sha512-uKjH4NZjiuKXVFfRhHSNEiPpwFb/P34hVpkg6HSFCJ2iynrtLTRNRLLAHbkXb1e2gMZT0cFVrKZuwC59xL13+g==", + "dependencies": { + "@anthropic-ai/sdk": "^0.4.2", + "@fortaine/fetch-event-source": "^3.0.6", + "browser-or-node": "^2.1.1", + "exponential-backoff": "^3.1.0", + "expr-eval": "^2.0.2", + "flat": "^5.0.2", + "is-binary-path": "^2.1.0", + "jsonpointer": "^5.0.1", + "object-hash": "^3.0.0", + "openai": "^3.2.0", + "p-queue": "^6.6.2", + "uuid": "^9.0.0", + "yaml": "^2.2.1", + "zod": "^3.21.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@dqbd/tiktoken": "^1.0.2", + "@getmetal/metal-sdk": "*", + "@huggingface/inference": "^1.5.1", + "@pinecone-database/pinecone": "^0.0.10", + "@supabase/supabase-js": "^2.10.0", + "cheerio": "^1.0.0-rc.12", + "chromadb": "^1.3.0", + "cohere-ai": "^5.0.2", + "d3-dsv": "^2.0.0", + "epub2": "^3.0.1", + "hnswlib-node": "^1.4.2", + "html-to-text": "^9.0.5", + "mammoth": "*", + "pdfjs-dist": "^3.4.120", + "puppeteer": "^19.7.2", + "redis": "^4.6.4", + "replicate": "^0.9.0", + "srt-parser-2": "^1.2.2", + "typeorm": "^0.3.12" + }, + "peerDependenciesMeta": { + "@dqbd/tiktoken": { + "optional": true + }, + "@getmetal/metal-sdk": { + "optional": true + }, + "@huggingface/inference": { + "optional": true + }, + "@pinecone-database/pinecone": { + "optional": true + }, + "@supabase/supabase-js": { + "optional": true + }, + "cheerio": { + "optional": true + }, + "chromadb": { + "optional": true + }, + "cohere-ai": { + "optional": true + }, + "d3-dsv": { + "optional": true + }, + "epub2": { + "optional": true + }, + "hnswlib-node": { + "optional": true + }, + "html-to-text": { + "optional": true + }, + "mammoth": { + "optional": true + }, + "pdfjs-dist": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "redis": { + "optional": true + }, + "replicate": { + "optional": true + }, + "srt-parser-2": { + "optional": true + }, + "typeorm": { + "optional": true + } + } + }, + "node_modules/langchain/node_modules/yaml": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", + "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "engines": { + "node": ">= 14" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -2968,6 +3241,25 @@ "node": ">=10.0.0" } }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/min-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", @@ -3213,7 +3505,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "dev": true, "engines": { "node": ">= 6" } @@ -3227,6 +3518,15 @@ "wrappy": "1" } }, + "node_modules/openai": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz", + "integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==", + "dependencies": { + "axios": "^0.26.0", + "form-data": "^4.0.0" + } + }, "node_modules/optionator": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", @@ -3244,6 +3544,14 @@ "node": ">= 0.8.0" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -3274,6 +3582,32 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -4309,8 +4643,7 @@ "node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "node_modules/ts-interface-checker": { "version": "0.1.13", @@ -4444,6 +4777,14 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vite": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/vite/-/vite-4.2.1.tgz", @@ -4565,14 +4906,12 @@ "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -4643,9 +4982,31 @@ "funding": { "url": "https://github.com/sponsors/sindresorhus" } + }, + "node_modules/zod": { + "version": "3.21.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", + "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } } }, "dependencies": { + "@anthropic-ai/sdk": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.4.3.tgz", + "integrity": "sha512-SZrlXvjUUYT9rPmSzlTtmVk1OjVNpkCzILRluhiYwNcxXfQyvPJDi0CI6PyymygcgtqEF5EVqhKmC/PtPsNEIw==", + "requires": { + "@fortaine/fetch-event-source": "^3.0.6", + "cross-fetch": "^3.1.5" + } + }, + "@dqbd/tiktoken": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.4.tgz", + "integrity": "sha512-C0HrJj2RNlsB3wslfNHGNH8xN7QQMki+y4JkUor/GE+oIfPvH7yVep9l1/2powam8AAH6+gdv5MggA5gsszweg==" + }, "@esbuild/android-arm": { "version": "0.17.15", "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.15.tgz", @@ -4838,6 +5199,11 @@ "integrity": "sha512-IoD2MfUnOV58ghIHCiil01PcohxjbYR/qCxsoC+xNgUwh1EY8jOOrYmu3d3a71+tJJ23uscEV4X2HJWMsPJu4g==", "dev": true }, + "@fortaine/fetch-event-source": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@fortaine/fetch-event-source/-/fetch-event-source-3.0.6.tgz", + "integrity": "sha512-621GAuLMvKtyZQ3IA6nlDWhV1V/7PGOTNIGLUifxt0KzM+dZIweJ6F3XvQF3QnqeNfS1N7WQ0Kil1Di/lhChEw==" + }, "@humanwhocodes/config-array": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.8.tgz", @@ -5525,6 +5891,11 @@ "integrity": "sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==", "dev": true }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, "autoprefixer": { "version": "10.4.14", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.14.tgz", @@ -5539,6 +5910,14 @@ "postcss-value-parser": "^4.2.0" } }, + "axios": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "requires": { + "follow-redirects": "^1.14.8" + } + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -5548,8 +5927,7 @@ "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "dev": true + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==" }, "bindings": { "version": "1.5.0", @@ -5579,6 +5957,11 @@ "fill-range": "^7.0.1" } }, + "browser-or-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", + "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" + }, "browserslist": { "version": "4.21.5", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", @@ -5709,6 +6092,14 @@ "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", "dev": true }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "requires": { + "delayed-stream": "~1.0.0" + } + }, "commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -5733,6 +6124,24 @@ "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", "dev": true }, + "cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "requires": { + "node-fetch": "2.6.7" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + } + } + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -5780,6 +6189,11 @@ "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", "dev": true }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" + }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", @@ -5834,6 +6248,12 @@ "esutils": "^2.0.2" } }, + "dotenv": { + "version": "16.0.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", + "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "dev": true + }, "electron-to-chromium": { "version": "1.4.356", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.356.tgz", @@ -6059,6 +6479,21 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==" + }, + "exponential-backoff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", + "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==" + }, + "expr-eval": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/expr-eval/-/expr-eval-2.0.2.tgz", + "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==" + }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -6144,6 +6579,11 @@ "path-exists": "^4.0.0" } }, + "flat": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==" + }, "flat-cache": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.0.4.tgz", @@ -6160,6 +6600,21 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, + "follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + }, + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "fraction.js": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", @@ -6377,7 +6832,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, "requires": { "binary-extensions": "^2.0.0" } @@ -6463,12 +6917,45 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==" + }, "kleur": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.5.tgz", "integrity": "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==", "dev": true }, + "langchain": { + "version": "0.0.51", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.51.tgz", + "integrity": "sha512-uKjH4NZjiuKXVFfRhHSNEiPpwFb/P34hVpkg6HSFCJ2iynrtLTRNRLLAHbkXb1e2gMZT0cFVrKZuwC59xL13+g==", + "requires": { + "@anthropic-ai/sdk": "^0.4.2", + "@fortaine/fetch-event-source": "^3.0.6", + "browser-or-node": "^2.1.1", + "exponential-backoff": "^3.1.0", + "expr-eval": "^2.0.2", + "flat": "^5.0.2", + "is-binary-path": "^2.1.0", + "jsonpointer": "^5.0.1", + "object-hash": "^3.0.0", + "openai": "^3.2.0", + "p-queue": "^6.6.2", + "uuid": "^9.0.0", + "yaml": "^2.2.1", + "zod": "^3.21.4" + }, + "dependencies": { + "yaml": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", + "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==" + } + } + }, "levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -6590,6 +7077,19 @@ "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", "dev": true }, + "mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" + }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "requires": { + "mime-db": "1.52.0" + } + }, "min-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", @@ -6763,8 +7263,7 @@ "object-hash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-3.0.0.tgz", - "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==", - "dev": true + "integrity": "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==" }, "once": { "version": "1.4.0", @@ -6775,6 +7274,15 @@ "wrappy": "1" } }, + "openai": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz", + "integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==", + "requires": { + "axios": "^0.26.0", + "form-data": "^4.0.0" + } + }, "optionator": { "version": "0.9.1", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", @@ -6789,6 +7297,11 @@ "word-wrap": "^1.2.3" } }, + "p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==" + }, "p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -6807,6 +7320,23 @@ "p-limit": "^3.0.2" } }, + "p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "requires": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + } + }, + "p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "requires": { + "p-finally": "^1.0.0" + } + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7478,8 +8008,7 @@ "tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "ts-interface-checker": { "version": "0.1.13", @@ -7571,6 +8100,11 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==" + }, "vite": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/vite/-/vite-4.2.1.tgz", @@ -7616,14 +8150,12 @@ "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "dev": true + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, "requires": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" @@ -7676,6 +8208,11 @@ "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", "dev": true + }, + "zod": { + "version": "3.21.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", + "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==" } } } diff --git a/package.json b/package.json index 0f0608f..0bc2e48 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "@typescript-eslint/eslint-plugin": "^5.45.0", "@typescript-eslint/parser": "^5.45.0", "autoprefixer": "^10.4.14", + "dotenv": "^16.0.3", "eslint": "^8.28.0", "eslint-config-prettier": "^8.5.0", "eslint-plugin-svelte3": "^4.0.0", @@ -38,5 +39,9 @@ "vite": "^4.2.0", "vitest": "^0.25.3" }, - "type": "module" + "type": "module", + "dependencies": { + "@dqbd/tiktoken": "^1.0.4", + "langchain": "^0.0.51" + } } diff --git a/src/lib/env.ts b/src/lib/env.ts new file mode 100644 index 0000000..2802599 --- /dev/null +++ b/src/lib/env.ts @@ -0,0 +1,5 @@ +import * as dotenv from 'dotenv'; +const config = dotenv.config(); +const OPENAI_API_KEY = config.OPENAI_API_KEY; + +export { OPENAI_API_KEY }; diff --git a/src/lib/shared/stores/prompt.ts b/src/lib/shared/stores/prompt.ts deleted file mode 100644 index 196f724..0000000 --- a/src/lib/shared/stores/prompt.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { writable } from 'svelte/store'; - -const prompt = writable(JSON.stringify([])); - -export default prompt; diff --git a/src/lib/shared/stores/summary.ts b/src/lib/shared/stores/summary.ts new file mode 100644 index 0000000..b337967 --- /dev/null +++ b/src/lib/shared/stores/summary.ts @@ -0,0 +1,5 @@ +import { writable } from 'svelte/store'; + +const summary = writable(JSON.stringify([])); + +export default summary; diff --git a/src/lib/types/env.d.ts b/src/lib/types/env.d.ts new file mode 100644 index 0000000..7769357 --- /dev/null +++ b/src/lib/types/env.d.ts @@ -0,0 +1,3 @@ +declare module 'dotenv' { + function config(options?: DotenvParseOptions): { [name: string]: string }; +} diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index 43d7b7e..72aedde 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -1,6 +1,6 @@ diff --git a/src/routes/+server.ts b/src/routes/+server.ts index e224903..ecf1477 100644 --- a/src/routes/+server.ts +++ b/src/routes/+server.ts @@ -1,35 +1,37 @@ import { fail } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; - -const parseFile = async (file: Blob) => { - const fileSize = file.size; - const chunkSize = 4000; - const chunks = Math.ceil(fileSize / chunkSize); - const convoArray: string[] = []; - const decoder = new TextDecoder(); - - for (let i = 0; i < chunks; i++) { - const start = i * chunkSize; - const end = Math.min(start + chunkSize, fileSize); - const buffer = await file.slice(start, end).arrayBuffer(); - const text = decoder.decode(buffer); - convoArray.push(text); - } - return convoArray; -}; +import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; +import { OpenAI } from 'langchain/llms'; +import { loadSummarizationChain } from 'langchain/chains'; export const POST = (async ({ request, url }) => { try { - const data = Object.fromEntries(await request.formData()); - const file = data.file as Blob; - const conversation = await parseFile(file); + const form = await request.formData(); - return new Response(JSON.stringify(conversation), { + const file = form.get('file'); + if (!file) throw new Error('No prompt found in the request.'); + const text = await (file as Blob).text(); + + const model = new OpenAI({ temperature: 0 }); + const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 }); + const docs = await textSplitter.createDocuments([text]); + + // This convenience function creates a document chain prompted to summarize a set of documents. + const chain = loadSummarizationChain(model); + const response = await chain.call({ + input_documents: docs + }); + + return new Response(JSON.stringify({ response: response.text }), { headers: { 'Content-Type': 'application/json' } }); } catch (err) { + console.log(err); + + const appError = err as App.Error; + console.log(appError.message); throw fail(500, { err: err }); } }) satisfies RequestHandler; diff --git a/src/routes/about/+page.svelte b/src/routes/about/+page.svelte index fd352ee..764052c 100644 --- a/src/routes/about/+page.svelte +++ b/src/routes/about/+page.svelte @@ -3,11 +3,25 @@ >

what dis?

- ChatGPT has a limit of ~4k characters per prompt, so to ask questions about a large body of text - would require that you split the text in 4k chunks. + ChatGPT has a character limit per prompt, so to ask questions about a large body of text would + require that you split the text and ask for ChatGPT to wait until the last prompt with the text + before generating a summary. +

+

+ ChatGPT's web interface also can't be adjusted to lower the probability of the AI spouting + unrelated nonsense. The technical term is "temperature", and the higher the temperature, the + more randomness in the output.

- With Sumi, you can simply upload the entire document as a text file, and it will handle - everything else. + With Sumi, you can simply enter a URL or upload the entire document as a text file, and it will + handle everything else.

+ + Source code + diff --git a/src/routes/chat/+page.svelte b/src/routes/chat/+page.svelte index 7a9940f..0877463 100644 --- a/src/routes/chat/+page.svelte +++ b/src/routes/chat/+page.svelte @@ -1,14 +1,16 @@
-
WIP
+

Summary

+
{currentSummary}