allow paths other than /v1

Author: j4ys0n

README.md

@@ -2,7 +2,7 @@
 
 Manages Nginx for reverse proxy to multiple LLMs, with TLS & Bearer Auth tokens. Deployed with docker.
 
-- Aggregates multiple OpenAI-type LLM APIs (all routes must be prefixed with "/v1")
+- Aggregates multiple OpenAI-type LLM APIs
 - Supports cloudflare domains
 - Uses Let's Encrypt for TLS certificates
 - Uses certbot for certificate issuance and renewal
@@ -33,7 +33,7 @@ version: '3.6'
 
 services:
   llmp:
-    image: ghcr.io/j4ys0n/llm-proxy:1.5.0
+    image: ghcr.io/j4ys0n/llm-proxy:1.5.1
     container_name: llmp
     hostname: llmp
     restart: unless-stopped
@@ -51,7 +51,7 @@ services:
 Here's what your `.env` file should look like:
 ```bash
 PORT=8080 # node.js listen port. right now nginx is hard coded, so don't change this.
-TARGET_URLS=http://localhost:1234,http://192.168.1.100:1234|api-key-here # list of api endpoints (don't add /v1)
+TARGET_URLS=http://localhost:1234,http://192.168.1.100:1234|api-key-here # list of api endpoints (/v1 is optional)
 JWT_SECRET=randomly_generated_secret # secret for JWT token generation, change this!
 AUTH_USERNAME=admin
 AUTH_PASSWORD=secure_password # super basic auth credentials for the admin interface

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.6'
 
 services:
   llmp:
-    image: ghcr.io/j4ys0n/llm-proxy:1.5.0
+    image: ghcr.io/j4ys0n/llm-proxy:1.5.1
     container_name: llmp
     hostname: llmp
     restart: unless-stopped

package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-proxy",
-  "version": "1.5.0",
+  "version": "1.5.1",
   "description": "Manages Nginx for reverse proxy to multiple LLMs, with TLS & Bearer Auth tokens",
   "main": "dist/index.js",
   "scripts": {

src/controllers/llm.ts

@@ -15,19 +15,43 @@ export interface ModelMap {
 
 const defaultContentType = 'application/json'
 
+function getPath(url: string): { path: string, base: string, apiKey?: string } {
+  try {
+    const urlParts = url.split('|')
+    const apiKey = urlParts.length > 1 ? urlParts[1] : undefined
+    const urlObject = new URL(url)
+    return {
+      path: urlObject.pathname || '/v1',
+      base: urlObject.origin,
+      apiKey
+    }
+  } catch (error) {
+    // Return the input if it's already a path starting with '/'
+    if (url.startsWith('/')) return { path: url, base: 'http://localhost' }
+    // Return '/v1' for invalid URLs
+    return { path: '/v1', base: 'http://localhost' }
+  }
+}
+
 async function fetchModels(targetUrls: string[]): Promise<ModelMap> {
   const tmp: ModelMap = {}
   for (const urlAndToken of targetUrls) {
     const [url, apiKey] = urlAndToken.split('|').map(s => s.trim())
-    const reqHeaders: { [key: string]: string } = {
+    const { path, base } = getPath(url)
+    const headers: { [key: string]: string } = {
       accept: defaultContentType,
       'Content-Type': defaultContentType
     }
     if (apiKey != null && apiKey !== '') {
-      reqHeaders['Authorization'] = `Bearer ${apiKey}`
+      headers['Authorization'] = `Bearer ${apiKey}`
+    }
+    const params = {
+      method: 'GET',
+      url: `${base}/${path}/models`,
+      headers
     }
     try {
-      const response = await axios.get(`${url}/v1/models`)
+      const response = await axios(params)
       const models = response.data.data || []
       const hostId = extractDomainName(url)
       models.forEach((model: Model) => {
@@ -85,26 +109,36 @@ export class LLMController {
   public async forwardPostRequest(req: Request, res: Response, next: NextFunction) {
     if (
       req.method === 'POST' &&
-      (req.path.startsWith('v1/') || req.path.startsWith('/v1/')) &&
+      (req.path.startsWith('v1') || req.path.startsWith('/v1')) &&
       req.body != null &&
       req.body.model != null &&
       this.targetUrls.length > 0
     ) {
       const { model: modelId } = req.body
-      let targetUrl = this.targetUrls[0] // Default to first URL if no matching model found
+      const { base: firstBaseUrl, path: firstPath, apiKey: firstApiKey } = getPath(this.targetUrls[0])
+      let targetUrl = firstBaseUrl // Default to first URL if no matching model found
+      let targetPath = firstPath
+      let targetApiKey = firstApiKey
 
       const hash = md5(modelId)
       if (modelId && this.modelCache[hash]) {
-        targetUrl = this.modelCache[hash].url
+        const { path, base, apiKey } = getPath(this.modelCache[hash].url)
+        targetUrl = base
+        targetPath = path
+        targetApiKey = apiKey
       }
-      const fullUrl = new URL(req.path, targetUrl).toString()
+      const reqPath = req.path.startsWith('/v1/') ? req.path.replace('/v1', targetPath) : `${targetPath}${req.path}`
+      const fullUrl = new URL(reqPath, targetUrl).toString()
       log('info', `Forwarding request to: ${fullUrl} -> ${modelId}`)
-
+      const headers = { ...req.headers }
+      if (targetApiKey) {
+        headers['Authorization'] = `Bearer ${targetApiKey}`
+      }
       try {
         const axiosConfig: AxiosRequestConfig = {
           method: req.method,
           url: fullUrl,
-          headers: { ...req.headers },
+          headers,
           data: req.body,
           responseType: 'stream'
         }