Merge pull request #12 from j4ys0n/request-limit

Request limit

Author: j4ys0n

Dockerfile

@@ -1,5 +1,5 @@
 # Use an official Node.js runtime as the base image
-FROM node:18.20.4-bullseye
+FROM node:18.20.3-bullseye
 
 # Install Nginx
 RUN apt-get update && apt-get install -y nginx certbot python3-certbot-dns-cloudflare

package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-proxy",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "description": "Manages Nginx for reverse proxy to multiple LLMs, with TLS & Bearer Auth tokens",
   "main": "dist/index.js",
   "scripts": {

src/controllers/llm.ts

@@ -1,5 +1,5 @@
 import { Express, NextFunction, Request, RequestHandler, Response } from 'express'
-import { log, md5, sleep } from '../utils/general'
+import { log, md5, sleep, extractDomainName } from '../utils/general'
 import axios, { AxiosRequestConfig } from 'axios'
 
 export interface Model {
@@ -19,7 +19,7 @@ async function fetchModels(targetUrls: string[]): Promise<ModelMap> {
     try {
       const response = await axios.get(`${url}/v1/models`)
       const models = response.data.data || []
-
+      const hostId = extractDomainName(url)
       models.forEach((model: Model) => {
         const hash = md5(model.id)
         tmp[hash] = { url, model }

src/index.ts

@@ -1,5 +1,6 @@
 import dotenv from 'dotenv'
 import express from 'express'
+import bodyParser from 'body-parser'
 import { NginxController } from './controllers/nginx'
 import { LLMController } from './controllers/llm'
 import { tokenMiddleware } from './utils/auth'
@@ -13,9 +14,15 @@ const targetUrls = (process.env.TARGET_URLS || 'http://example.com').split(',').
 
 app.use(express.json())
 
+const payloadLimit = process.env.PAYLOAD_LIMIT || '250kb'
+//support application/json type post data (default limit is 100kb)
+app.use(bodyParser.json({ limit: payloadLimit }))
+//support application/x-www-form-urlencoded post data (default limit is 100kb)
+app.use(bodyParser.urlencoded({ limit: payloadLimit, extended: false }))
+
 // Express routes
 app.get('/', (req, res) => {
-  res.send('LLM Aggregation Proxy')
+  res.send('LLM Proxy')
 })
 
 const authController = new AuthController({ app })

src/utils/general.ts

@@ -15,3 +15,7 @@ export function md5(data: Object | string): string {
 export function sleep(ms: number) {
   return new Promise((r) => setTimeout(r, ms))
 }
+
+export function extractDomainName(url: string) {
+  return url.replace('https://', '').replace('http://', '').split('/')[0].replace(/\./g, '_')
+}