feat: монорепо миграция, Discover/SearxNG улучшения
- Миграция на монорепозиторий (apps/frontend, apps/chat-service, etc.) - Discover: проверка SearxNG, понятное empty state при ненастроенном поиске - searxng.ts: валидация URL, проверка JSON-ответа, авто-добавление http:// - docker/searxng-config: настройки для JSON API SearxNG Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
home
11
apps/storage-service/drizzle.config.ts
Normal file
11
apps/storage-service/drizzle.config.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import { defineConfig } from 'drizzle-kit';
|
||||
import path from 'path';
|
||||
|
||||
export default defineConfig({
|
||||
dialect: 'sqlite',
|
||||
schema: './src/lib/db/schema.ts',
|
||||
out: './drizzle',
|
||||
dbCredentials: {
|
||||
url: path.join(process.cwd(), 'data', 'db.sqlite'),
|
||||
},
|
||||
});
|
||||
16
apps/storage-service/drizzle/0000_fuzzy_randall.sql
Normal file
16
apps/storage-service/drizzle/0000_fuzzy_randall.sql
Normal file
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE IF NOT EXISTS `chats` (
|
||||
`id` text PRIMARY KEY NOT NULL,
|
||||
`title` text NOT NULL,
|
||||
`createdAt` text NOT NULL,
|
||||
`focusMode` text NOT NULL,
|
||||
`files` text DEFAULT '[]'
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE IF NOT EXISTS `messages` (
|
||||
`id` integer PRIMARY KEY NOT NULL,
|
||||
`content` text NOT NULL,
|
||||
`chatId` text NOT NULL,
|
||||
`messageId` text NOT NULL,
|
||||
`type` text,
|
||||
`metadata` text
|
||||
);
|
||||
1
apps/storage-service/drizzle/0001_wise_rockslide.sql
Normal file
1
apps/storage-service/drizzle/0001_wise_rockslide.sql
Normal file
@@ -0,0 +1 @@
|
||||
/* Do nothing */
|
||||
1
apps/storage-service/drizzle/0002_daffy_wrecker.sql
Normal file
1
apps/storage-service/drizzle/0002_daffy_wrecker.sql
Normal file
@@ -0,0 +1 @@
|
||||
/* do nothing */
|
||||
116
apps/storage-service/drizzle/meta/0000_snapshot.json
Normal file
116
apps/storage-service/drizzle/meta/0000_snapshot.json
Normal file
@@ -0,0 +1,116 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "ef3a044b-0f34-40b5-babb-2bb3a909ba27",
|
||||
"prevId": "00000000-0000-0000-0000-000000000000",
|
||||
"tables": {
|
||||
"chats": {
|
||||
"name": "chats",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"title": {
|
||||
"name": "title",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"createdAt": {
|
||||
"name": "createdAt",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"focusMode": {
|
||||
"name": "focusMode",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"files": {
|
||||
"name": "files",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"messages": {
|
||||
"name": "messages",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "integer",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"content": {
|
||||
"name": "content",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"chatId": {
|
||||
"name": "chatId",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"messageId": {
|
||||
"name": "messageId",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"type": {
|
||||
"name": "type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"metadata": {
|
||||
"name": "metadata",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
125
apps/storage-service/drizzle/meta/0001_snapshot.json
Normal file
125
apps/storage-service/drizzle/meta/0001_snapshot.json
Normal file
@@ -0,0 +1,125 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "6dedf55f-0e44-478f-82cf-14a21ac686f8",
|
||||
"prevId": "ef3a044b-0f34-40b5-babb-2bb3a909ba27",
|
||||
"tables": {
|
||||
"chats": {
|
||||
"name": "chats",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"title": {
|
||||
"name": "title",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"createdAt": {
|
||||
"name": "createdAt",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"focusMode": {
|
||||
"name": "focusMode",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"files": {
|
||||
"name": "files",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"messages": {
|
||||
"name": "messages",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "integer",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"type": {
|
||||
"name": "type",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"chatId": {
|
||||
"name": "chatId",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"createdAt": {
|
||||
"name": "createdAt",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false,
|
||||
"default": "CURRENT_TIMESTAMP"
|
||||
},
|
||||
"messageId": {
|
||||
"name": "messageId",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"content": {
|
||||
"name": "content",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false
|
||||
},
|
||||
"sources": {
|
||||
"name": "sources",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
132
apps/storage-service/drizzle/meta/0002_snapshot.json
Normal file
132
apps/storage-service/drizzle/meta/0002_snapshot.json
Normal file
@@ -0,0 +1,132 @@
|
||||
{
|
||||
"version": "6",
|
||||
"dialect": "sqlite",
|
||||
"id": "1c5eb804-d6b4-48ec-9a8f-75fb729c8e52",
|
||||
"prevId": "6dedf55f-0e44-478f-82cf-14a21ac686f8",
|
||||
"tables": {
|
||||
"chats": {
|
||||
"name": "chats",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "text",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"title": {
|
||||
"name": "title",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"createdAt": {
|
||||
"name": "createdAt",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"sources": {
|
||||
"name": "sources",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"files": {
|
||||
"name": "files",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
},
|
||||
"messages": {
|
||||
"name": "messages",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "integer",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"messageId": {
|
||||
"name": "messageId",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"chatId": {
|
||||
"name": "chatId",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"backendId": {
|
||||
"name": "backendId",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"query": {
|
||||
"name": "query",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"createdAt": {
|
||||
"name": "createdAt",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"autoincrement": false
|
||||
},
|
||||
"responseBlocks": {
|
||||
"name": "responseBlocks",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": "'[]'"
|
||||
},
|
||||
"status": {
|
||||
"name": "status",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"autoincrement": false,
|
||||
"default": "'answering'"
|
||||
}
|
||||
},
|
||||
"indexes": {},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"checkConstraints": {}
|
||||
}
|
||||
},
|
||||
"views": {},
|
||||
"enums": {},
|
||||
"_meta": {
|
||||
"schemas": {},
|
||||
"tables": {},
|
||||
"columns": {}
|
||||
},
|
||||
"internal": {
|
||||
"indexes": {}
|
||||
}
|
||||
}
|
||||
27
apps/storage-service/drizzle/meta/_journal.json
Normal file
27
apps/storage-service/drizzle/meta/_journal.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"version": "7",
|
||||
"dialect": "sqlite",
|
||||
"entries": [
|
||||
{
|
||||
"idx": 0,
|
||||
"version": "6",
|
||||
"when": 1748405503809,
|
||||
"tag": "0000_fuzzy_randall",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "6",
|
||||
"when": 1758863991284,
|
||||
"tag": "0001_wise_rockslide",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 2,
|
||||
"version": "6",
|
||||
"when": 1763732708332,
|
||||
"tag": "0002_daffy_wrecker",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
29
apps/storage-service/src/lib/config/clientRegistry.ts
Normal file
29
apps/storage-service/src/lib/config/clientRegistry.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
'use client';
|
||||
|
||||
const getClientConfig = (key: string, defaultVal?: any) => {
|
||||
return localStorage.getItem(key) ?? defaultVal ?? undefined;
|
||||
};
|
||||
|
||||
export const getTheme = () => getClientConfig('theme', 'dark');
|
||||
|
||||
export const getAutoMediaSearch = () =>
|
||||
getClientConfig('autoMediaSearch', 'true') === 'true';
|
||||
|
||||
export const getSystemInstructions = () =>
|
||||
getClientConfig('systemInstructions', '');
|
||||
|
||||
export const getShowWeatherWidget = () =>
|
||||
getClientConfig('showWeatherWidget', 'true') === 'true';
|
||||
|
||||
export const getShowNewsWidget = () =>
|
||||
getClientConfig('showNewsWidget', 'true') === 'true';
|
||||
|
||||
export const getMeasurementUnit = () => {
|
||||
const value =
|
||||
getClientConfig('measureUnit') ??
|
||||
getClientConfig('measurementUnit', 'metric');
|
||||
|
||||
if (typeof value !== 'string') return 'metric';
|
||||
|
||||
return value.toLowerCase();
|
||||
};
|
||||
390
apps/storage-service/src/lib/config/index.ts
Normal file
390
apps/storage-service/src/lib/config/index.ts
Normal file
@@ -0,0 +1,390 @@
|
||||
import path from 'node:path';
|
||||
import fs from 'fs';
|
||||
import { Config, ConfigModelProvider, UIConfigSections } from './types';
|
||||
import { hashObj } from '../../../../shared-utils/src/serverUtils';
|
||||
import { getModelProvidersUIConfigSection } from '../models/providers';
|
||||
|
||||
class ConfigManager {
|
||||
configPath: string = path.join(
|
||||
process.env.DATA_DIR || process.cwd(),
|
||||
'/data/config.json',
|
||||
);
|
||||
configVersion = 1;
|
||||
currentConfig: Config = {
|
||||
version: this.configVersion,
|
||||
setupComplete: false,
|
||||
preferences: {},
|
||||
personalization: {},
|
||||
modelProviders: [],
|
||||
search: {
|
||||
searxngURL: '',
|
||||
},
|
||||
};
|
||||
uiConfigSections: UIConfigSections = {
|
||||
preferences: [
|
||||
{
|
||||
name: 'Theme',
|
||||
key: 'theme',
|
||||
type: 'select',
|
||||
options: [
|
||||
{
|
||||
name: 'Light',
|
||||
value: 'light',
|
||||
},
|
||||
{
|
||||
name: 'Dark',
|
||||
value: 'dark',
|
||||
},
|
||||
],
|
||||
required: false,
|
||||
description: 'Choose between light and dark layouts for the app.',
|
||||
default: 'dark',
|
||||
scope: 'client',
|
||||
},
|
||||
{
|
||||
name: 'Measurement Unit',
|
||||
key: 'measureUnit',
|
||||
type: 'select',
|
||||
options: [
|
||||
{
|
||||
name: 'Imperial',
|
||||
value: 'Imperial',
|
||||
},
|
||||
{
|
||||
name: 'Metric',
|
||||
value: 'Metric',
|
||||
},
|
||||
],
|
||||
required: false,
|
||||
description: 'Choose between Metric and Imperial measurement unit.',
|
||||
default: 'Metric',
|
||||
scope: 'client',
|
||||
},
|
||||
{
|
||||
name: 'Auto video & image search',
|
||||
key: 'autoMediaSearch',
|
||||
type: 'switch',
|
||||
required: false,
|
||||
description: 'Automatically search for relevant images and videos.',
|
||||
default: true,
|
||||
scope: 'client',
|
||||
},
|
||||
{
|
||||
name: 'Show weather widget',
|
||||
key: 'showWeatherWidget',
|
||||
type: 'switch',
|
||||
required: false,
|
||||
description: 'Display the weather card on the home screen.',
|
||||
default: true,
|
||||
scope: 'client',
|
||||
},
|
||||
{
|
||||
name: 'Show news widget',
|
||||
key: 'showNewsWidget',
|
||||
type: 'switch',
|
||||
required: false,
|
||||
description: 'Display the recent news card on the home screen.',
|
||||
default: true,
|
||||
scope: 'client',
|
||||
},
|
||||
],
|
||||
personalization: [
|
||||
{
|
||||
name: 'System Instructions',
|
||||
key: 'systemInstructions',
|
||||
type: 'textarea',
|
||||
required: false,
|
||||
description: 'Add custom behavior or tone for the model.',
|
||||
placeholder:
|
||||
'e.g., "Respond in a friendly and concise tone" or "Use British English and format answers as bullet points."',
|
||||
scope: 'client',
|
||||
},
|
||||
],
|
||||
modelProviders: [],
|
||||
search: [
|
||||
{
|
||||
name: 'SearXNG URL',
|
||||
key: 'searxngURL',
|
||||
type: 'string',
|
||||
required: false,
|
||||
description: 'The URL of your SearXNG instance',
|
||||
placeholder: 'http://localhost:4000',
|
||||
default: '',
|
||||
scope: 'server',
|
||||
env: 'SEARXNG_API_URL',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
constructor() {
|
||||
this.initialize();
|
||||
}
|
||||
|
||||
private initialize() {
|
||||
this.initializeConfig();
|
||||
this.initializeFromEnv();
|
||||
}
|
||||
|
||||
private saveConfig() {
|
||||
fs.writeFileSync(
|
||||
this.configPath,
|
||||
JSON.stringify(this.currentConfig, null, 2),
|
||||
);
|
||||
}
|
||||
|
||||
private initializeConfig() {
|
||||
const exists = fs.existsSync(this.configPath);
|
||||
if (!exists) {
|
||||
fs.writeFileSync(
|
||||
this.configPath,
|
||||
JSON.stringify(this.currentConfig, null, 2),
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
this.currentConfig = JSON.parse(
|
||||
fs.readFileSync(this.configPath, 'utf-8'),
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof SyntaxError) {
|
||||
console.error(
|
||||
`Error parsing config file at ${this.configPath}:`,
|
||||
err,
|
||||
);
|
||||
console.log(
|
||||
'Loading default config and overwriting the existing file.',
|
||||
);
|
||||
fs.writeFileSync(
|
||||
this.configPath,
|
||||
JSON.stringify(this.currentConfig, null, 2),
|
||||
);
|
||||
return;
|
||||
} else {
|
||||
console.log('Unknown error reading config file:', err);
|
||||
}
|
||||
}
|
||||
|
||||
this.currentConfig = this.migrateConfig(this.currentConfig);
|
||||
}
|
||||
}
|
||||
|
||||
private migrateConfig(config: Config): Config {
|
||||
/* TODO: Add migrations */
|
||||
return config;
|
||||
}
|
||||
|
||||
private initializeFromEnv() {
|
||||
/* providers section*/
|
||||
const providerConfigSections = getModelProvidersUIConfigSection();
|
||||
|
||||
this.uiConfigSections.modelProviders = providerConfigSections;
|
||||
|
||||
const newProviders: ConfigModelProvider[] = [];
|
||||
|
||||
providerConfigSections.forEach((provider) => {
|
||||
const newProvider: ConfigModelProvider & { required?: string[] } = {
|
||||
id: crypto.randomUUID(),
|
||||
name: `${provider.name}`,
|
||||
type: provider.key,
|
||||
chatModels: [],
|
||||
embeddingModels: [],
|
||||
config: {},
|
||||
required: [],
|
||||
hash: '',
|
||||
};
|
||||
|
||||
provider.fields.forEach((field) => {
|
||||
newProvider.config[field.key] =
|
||||
process.env[field.env!] ||
|
||||
field.default ||
|
||||
''; /* Env var must exist for providers */
|
||||
|
||||
if (field.required) newProvider.required?.push(field.key);
|
||||
});
|
||||
|
||||
let configured = true;
|
||||
|
||||
newProvider.required?.forEach((r) => {
|
||||
if (!newProvider.config[r]) {
|
||||
configured = false;
|
||||
}
|
||||
});
|
||||
|
||||
if (configured) {
|
||||
const hash = hashObj(newProvider.config);
|
||||
newProvider.hash = hash;
|
||||
delete newProvider.required;
|
||||
|
||||
const exists = this.currentConfig.modelProviders.find(
|
||||
(p) => p.hash === hash,
|
||||
);
|
||||
|
||||
if (!exists) {
|
||||
newProviders.push(newProvider);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
this.currentConfig.modelProviders.push(...newProviders);
|
||||
|
||||
/* search section */
|
||||
this.uiConfigSections.search.forEach((f) => {
|
||||
if (f.env && !this.currentConfig.search[f.key]) {
|
||||
this.currentConfig.search[f.key] =
|
||||
process.env[f.env] ?? f.default ?? '';
|
||||
}
|
||||
});
|
||||
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
public getConfig(key: string, defaultValue?: any): any {
|
||||
const nested = key.split('.');
|
||||
let obj: any = this.currentConfig;
|
||||
|
||||
for (let i = 0; i < nested.length; i++) {
|
||||
const part = nested[i];
|
||||
if (obj == null) return defaultValue;
|
||||
|
||||
obj = obj[part];
|
||||
}
|
||||
|
||||
return obj === undefined ? defaultValue : obj;
|
||||
}
|
||||
|
||||
public updateConfig(key: string, val: any) {
|
||||
const parts = key.split('.');
|
||||
if (parts.length === 0) return;
|
||||
|
||||
let target: any = this.currentConfig;
|
||||
for (let i = 0; i < parts.length - 1; i++) {
|
||||
const part = parts[i];
|
||||
if (target[part] === null || typeof target[part] !== 'object') {
|
||||
target[part] = {};
|
||||
}
|
||||
|
||||
target = target[part];
|
||||
}
|
||||
|
||||
const finalKey = parts[parts.length - 1];
|
||||
target[finalKey] = val;
|
||||
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
public addModelProvider(type: string, name: string, config: any) {
|
||||
const newModelProvider: ConfigModelProvider = {
|
||||
id: crypto.randomUUID(),
|
||||
name,
|
||||
type,
|
||||
config,
|
||||
chatModels: [],
|
||||
embeddingModels: [],
|
||||
hash: hashObj(config),
|
||||
};
|
||||
|
||||
this.currentConfig.modelProviders.push(newModelProvider);
|
||||
this.saveConfig();
|
||||
|
||||
return newModelProvider;
|
||||
}
|
||||
|
||||
public removeModelProvider(id: string) {
|
||||
const index = this.currentConfig.modelProviders.findIndex(
|
||||
(p) => p.id === id,
|
||||
);
|
||||
|
||||
if (index === -1) return;
|
||||
|
||||
this.currentConfig.modelProviders =
|
||||
this.currentConfig.modelProviders.filter((p) => p.id !== id);
|
||||
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
public async updateModelProvider(id: string, name: string, config: any) {
|
||||
const provider = this.currentConfig.modelProviders.find((p) => {
|
||||
return p.id === id;
|
||||
});
|
||||
|
||||
if (!provider) throw new Error('Provider not found');
|
||||
|
||||
provider.name = name;
|
||||
provider.config = config;
|
||||
|
||||
this.saveConfig();
|
||||
|
||||
return provider;
|
||||
}
|
||||
|
||||
public addProviderModel(
|
||||
providerId: string,
|
||||
type: 'embedding' | 'chat',
|
||||
model: any,
|
||||
) {
|
||||
const provider = this.currentConfig.modelProviders.find(
|
||||
(p) => p.id === providerId,
|
||||
);
|
||||
|
||||
if (!provider) throw new Error('Invalid provider id');
|
||||
|
||||
delete model.type;
|
||||
|
||||
if (type === 'chat') {
|
||||
provider.chatModels.push(model);
|
||||
} else {
|
||||
provider.embeddingModels.push(model);
|
||||
}
|
||||
|
||||
this.saveConfig();
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
public removeProviderModel(
|
||||
providerId: string,
|
||||
type: 'embedding' | 'chat',
|
||||
modelKey: string,
|
||||
) {
|
||||
const provider = this.currentConfig.modelProviders.find(
|
||||
(p) => p.id === providerId,
|
||||
);
|
||||
|
||||
if (!provider) throw new Error('Invalid provider id');
|
||||
|
||||
if (type === 'chat') {
|
||||
provider.chatModels = provider.chatModels.filter(
|
||||
(m) => m.key !== modelKey,
|
||||
);
|
||||
} else {
|
||||
provider.embeddingModels = provider.embeddingModels.filter(
|
||||
(m) => m.key != modelKey,
|
||||
);
|
||||
}
|
||||
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
public isSetupComplete() {
|
||||
return this.currentConfig.setupComplete;
|
||||
}
|
||||
|
||||
public markSetupComplete() {
|
||||
if (!this.currentConfig.setupComplete) {
|
||||
this.currentConfig.setupComplete = true;
|
||||
}
|
||||
|
||||
this.saveConfig();
|
||||
}
|
||||
|
||||
public getUIConfigSections(): UIConfigSections {
|
||||
return this.uiConfigSections;
|
||||
}
|
||||
|
||||
public getCurrentConfig(): Config {
|
||||
return JSON.parse(JSON.stringify(this.currentConfig));
|
||||
}
|
||||
}
|
||||
|
||||
const configManager = new ConfigManager();
|
||||
|
||||
export default configManager;
|
||||
15
apps/storage-service/src/lib/config/serverRegistry.ts
Normal file
15
apps/storage-service/src/lib/config/serverRegistry.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import configManager from './index';
|
||||
import { ConfigModelProvider } from './types';
|
||||
|
||||
export const getConfiguredModelProviders = (): ConfigModelProvider[] => {
|
||||
return configManager.getConfig('modelProviders', []);
|
||||
};
|
||||
|
||||
export const getConfiguredModelProviderById = (
|
||||
id: string,
|
||||
): ConfigModelProvider | undefined => {
|
||||
return getConfiguredModelProviders().find((p) => p.id === id) ?? undefined;
|
||||
};
|
||||
|
||||
export const getSearxngURL = () =>
|
||||
configManager.getConfig('search.searxngURL', '');
|
||||
109
apps/storage-service/src/lib/config/types.ts
Normal file
109
apps/storage-service/src/lib/config/types.ts
Normal file
@@ -0,0 +1,109 @@
|
||||
import { Model } from '../models/types';
|
||||
|
||||
type BaseUIConfigField = {
|
||||
name: string;
|
||||
key: string;
|
||||
required: boolean;
|
||||
description: string;
|
||||
scope: 'client' | 'server';
|
||||
env?: string;
|
||||
};
|
||||
|
||||
type StringUIConfigField = BaseUIConfigField & {
|
||||
type: 'string';
|
||||
placeholder?: string;
|
||||
default?: string;
|
||||
};
|
||||
|
||||
type SelectUIConfigFieldOptions = {
|
||||
name: string;
|
||||
value: string;
|
||||
};
|
||||
|
||||
type SelectUIConfigField = BaseUIConfigField & {
|
||||
type: 'select';
|
||||
default?: string;
|
||||
options: SelectUIConfigFieldOptions[];
|
||||
};
|
||||
|
||||
type PasswordUIConfigField = BaseUIConfigField & {
|
||||
type: 'password';
|
||||
placeholder?: string;
|
||||
default?: string;
|
||||
};
|
||||
|
||||
type TextareaUIConfigField = BaseUIConfigField & {
|
||||
type: 'textarea';
|
||||
placeholder?: string;
|
||||
default?: string;
|
||||
};
|
||||
|
||||
type SwitchUIConfigField = BaseUIConfigField & {
|
||||
type: 'switch';
|
||||
default?: boolean;
|
||||
};
|
||||
|
||||
type UIConfigField =
|
||||
| StringUIConfigField
|
||||
| SelectUIConfigField
|
||||
| PasswordUIConfigField
|
||||
| TextareaUIConfigField
|
||||
| SwitchUIConfigField;
|
||||
|
||||
type ConfigModelProvider = {
|
||||
id: string;
|
||||
name: string;
|
||||
type: string;
|
||||
chatModels: Model[];
|
||||
embeddingModels: Model[];
|
||||
config: { [key: string]: any };
|
||||
hash: string;
|
||||
};
|
||||
|
||||
type Config = {
|
||||
version: number;
|
||||
setupComplete: boolean;
|
||||
preferences: {
|
||||
[key: string]: any;
|
||||
};
|
||||
personalization: {
|
||||
[key: string]: any;
|
||||
};
|
||||
modelProviders: ConfigModelProvider[];
|
||||
search: {
|
||||
[key: string]: any;
|
||||
};
|
||||
};
|
||||
|
||||
type EnvMap = {
|
||||
[key: string]: {
|
||||
fieldKey: string;
|
||||
providerKey: string;
|
||||
};
|
||||
};
|
||||
|
||||
type ModelProviderUISection = {
|
||||
name: string;
|
||||
key: string;
|
||||
fields: UIConfigField[];
|
||||
};
|
||||
|
||||
type UIConfigSections = {
|
||||
preferences: UIConfigField[];
|
||||
personalization: UIConfigField[];
|
||||
modelProviders: ModelProviderUISection[];
|
||||
search: UIConfigField[];
|
||||
};
|
||||
|
||||
export type {
|
||||
UIConfigField,
|
||||
Config,
|
||||
EnvMap,
|
||||
UIConfigSections,
|
||||
SelectUIConfigField,
|
||||
StringUIConfigField,
|
||||
ModelProviderUISection,
|
||||
ConfigModelProvider,
|
||||
TextareaUIConfigField,
|
||||
SwitchUIConfigField,
|
||||
};
|
||||
12
apps/storage-service/src/lib/db/index.ts
Normal file
12
apps/storage-service/src/lib/db/index.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import { drizzle } from 'drizzle-orm/better-sqlite3';
|
||||
import Database from 'better-sqlite3';
|
||||
import * as schema from './schema';
|
||||
import path from 'path';
|
||||
|
||||
const DATA_DIR = process.env.DATA_DIR || process.cwd();
|
||||
const sqlite = new Database(path.join(DATA_DIR, './data/db.sqlite'));
|
||||
const db = drizzle(sqlite, {
|
||||
schema: schema,
|
||||
});
|
||||
|
||||
export default db;
|
||||
288
apps/storage-service/src/lib/db/migrate.ts
Normal file
288
apps/storage-service/src/lib/db/migrate.ts
Normal file
@@ -0,0 +1,288 @@
|
||||
import Database from 'better-sqlite3';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
const DATA_DIR = process.env.DATA_DIR || process.cwd();
|
||||
const dbPath = path.join(DATA_DIR, './data/db.sqlite');
|
||||
|
||||
const db = new Database(dbPath);
|
||||
|
||||
const migrationsFolder = path.join(DATA_DIR, 'drizzle');
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS ran_migrations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
run_on DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`);
|
||||
|
||||
function sanitizeSql(content: string) {
|
||||
const statements = content
|
||||
.split(/--> statement-breakpoint/g)
|
||||
.map((stmt) =>
|
||||
stmt
|
||||
.split(/\r?\n/)
|
||||
.filter((l) => !l.trim().startsWith('-->'))
|
||||
.join('\n')
|
||||
.trim(),
|
||||
)
|
||||
.filter((stmt) => stmt.length > 0);
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
fs.readdirSync(migrationsFolder)
|
||||
.filter((f) => f.endsWith('.sql'))
|
||||
.sort()
|
||||
.forEach((file) => {
|
||||
const filePath = path.join(migrationsFolder, file);
|
||||
let content = fs.readFileSync(filePath, 'utf-8');
|
||||
const statements = sanitizeSql(content);
|
||||
|
||||
const migrationName = file.split('_')[0] || file;
|
||||
|
||||
const already = db
|
||||
.prepare('SELECT 1 FROM ran_migrations WHERE name = ?')
|
||||
.get(migrationName);
|
||||
|
||||
if (already) {
|
||||
console.log(`Skipping already-applied migration: ${file}`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (migrationName === '0001') {
|
||||
const messages = db
|
||||
.prepare(
|
||||
'SELECT id, type, metadata, content, chatId, messageId FROM messages',
|
||||
)
|
||||
.all();
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS messages_with_sources (
|
||||
id INTEGER PRIMARY KEY,
|
||||
type TEXT NOT NULL,
|
||||
chatId TEXT NOT NULL,
|
||||
createdAt TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
messageId TEXT NOT NULL,
|
||||
content TEXT,
|
||||
sources TEXT DEFAULT '[]'
|
||||
);
|
||||
`);
|
||||
|
||||
const insertMessage = db.prepare(`
|
||||
INSERT INTO messages_with_sources (type, chatId, createdAt, messageId, content, sources)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
messages.forEach((msg: any) => {
|
||||
while (typeof msg.metadata === 'string') {
|
||||
msg.metadata = JSON.parse(msg.metadata || '{}');
|
||||
}
|
||||
if (msg.type === 'user') {
|
||||
insertMessage.run(
|
||||
'user',
|
||||
msg.chatId,
|
||||
msg.metadata['createdAt'],
|
||||
msg.messageId,
|
||||
msg.content,
|
||||
'[]',
|
||||
);
|
||||
} else if (msg.type === 'assistant') {
|
||||
insertMessage.run(
|
||||
'assistant',
|
||||
msg.chatId,
|
||||
msg.metadata['createdAt'],
|
||||
msg.messageId,
|
||||
msg.content,
|
||||
'[]',
|
||||
);
|
||||
const sources = msg.metadata['sources'] || '[]';
|
||||
if (sources && sources.length > 0) {
|
||||
insertMessage.run(
|
||||
'source',
|
||||
msg.chatId,
|
||||
msg.metadata['createdAt'],
|
||||
`${msg.messageId}-source`,
|
||||
'',
|
||||
JSON.stringify(sources),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
db.exec('DROP TABLE messages;');
|
||||
db.exec('ALTER TABLE messages_with_sources RENAME TO messages;');
|
||||
} else if (migrationName === '0002') {
|
||||
/* Migrate chat */
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS chats_new (
|
||||
id TEXT PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
createdAt TEXT NOT NULL,
|
||||
sources TEXT DEFAULT '[]',
|
||||
files TEXT DEFAULT '[]'
|
||||
);
|
||||
`);
|
||||
|
||||
const chats = db
|
||||
.prepare('SELECT id, title, createdAt, files FROM chats')
|
||||
.all();
|
||||
|
||||
const insertChat = db.prepare(`
|
||||
INSERT INTO chats_new (id, title, createdAt, sources, files)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
chats.forEach((chat: any) => {
|
||||
let files = chat.files;
|
||||
while (typeof files === 'string') {
|
||||
files = JSON.parse(files || '[]');
|
||||
}
|
||||
|
||||
insertChat.run(
|
||||
chat.id,
|
||||
chat.title,
|
||||
chat.createdAt,
|
||||
'["web"]',
|
||||
JSON.stringify(files),
|
||||
);
|
||||
});
|
||||
|
||||
db.exec('DROP TABLE chats;');
|
||||
db.exec('ALTER TABLE chats_new RENAME TO chats;');
|
||||
|
||||
/* Migrate messages */
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS messages_new (
|
||||
id INTEGER PRIMARY KEY,
|
||||
messageId TEXT NOT NULL,
|
||||
chatId TEXT NOT NULL,
|
||||
backendId TEXT NOT NULL,
|
||||
query TEXT NOT NULL,
|
||||
createdAt TEXT NOT NULL,
|
||||
responseBlocks TEXT DEFAULT '[]',
|
||||
status TEXT DEFAULT 'answering'
|
||||
);
|
||||
`);
|
||||
|
||||
const messages = db
|
||||
.prepare(
|
||||
'SELECT id, messageId, chatId, type, content, createdAt, sources FROM messages ORDER BY id ASC',
|
||||
)
|
||||
.all();
|
||||
|
||||
const insertMessage = db.prepare(`
|
||||
INSERT INTO messages_new (messageId, chatId, backendId, query, createdAt, responseBlocks, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
|
||||
let currentMessageData: {
|
||||
sources?: any[];
|
||||
response?: string;
|
||||
query?: string;
|
||||
messageId?: string;
|
||||
chatId?: string;
|
||||
createdAt?: string;
|
||||
} = {};
|
||||
let lastCompleted = true;
|
||||
|
||||
messages.forEach((msg: any) => {
|
||||
if (msg.type === 'user' && lastCompleted) {
|
||||
currentMessageData = {};
|
||||
currentMessageData.messageId = msg.messageId;
|
||||
currentMessageData.chatId = msg.chatId;
|
||||
currentMessageData.query = msg.content;
|
||||
currentMessageData.createdAt = msg.createdAt;
|
||||
lastCompleted = false;
|
||||
} else if (msg.type === 'source' && !lastCompleted) {
|
||||
let sources = msg.sources;
|
||||
|
||||
while (typeof sources === 'string') {
|
||||
sources = JSON.parse(sources || '[]');
|
||||
}
|
||||
|
||||
currentMessageData.sources = sources;
|
||||
} else if (msg.type === 'assistant' && !lastCompleted) {
|
||||
currentMessageData.response = msg.content;
|
||||
insertMessage.run(
|
||||
currentMessageData.messageId,
|
||||
currentMessageData.chatId,
|
||||
`${currentMessageData.messageId}-backend`,
|
||||
currentMessageData.query,
|
||||
currentMessageData.createdAt,
|
||||
JSON.stringify([
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
type: 'text',
|
||||
data: currentMessageData.response || '',
|
||||
},
|
||||
...(currentMessageData.sources &&
|
||||
currentMessageData.sources.length > 0
|
||||
? [
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
type: 'source',
|
||||
data: currentMessageData.sources,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]),
|
||||
'completed',
|
||||
);
|
||||
|
||||
lastCompleted = true;
|
||||
} else if (msg.type === 'user' && !lastCompleted) {
|
||||
/* Message wasn't completed so we'll just create the record with empty response */
|
||||
insertMessage.run(
|
||||
currentMessageData.messageId,
|
||||
currentMessageData.chatId,
|
||||
`${currentMessageData.messageId}-backend`,
|
||||
currentMessageData.query,
|
||||
currentMessageData.createdAt,
|
||||
JSON.stringify([
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
type: 'text',
|
||||
data: '',
|
||||
},
|
||||
...(currentMessageData.sources &&
|
||||
currentMessageData.sources.length > 0
|
||||
? [
|
||||
{
|
||||
id: crypto.randomUUID(),
|
||||
type: 'source',
|
||||
data: currentMessageData.sources,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
]),
|
||||
'completed',
|
||||
);
|
||||
|
||||
lastCompleted = true;
|
||||
}
|
||||
});
|
||||
|
||||
db.exec('DROP TABLE messages;');
|
||||
db.exec('ALTER TABLE messages_new RENAME TO messages;');
|
||||
} else {
|
||||
// Execute each statement separately
|
||||
statements.forEach((stmt) => {
|
||||
if (stmt.trim()) {
|
||||
db.exec(stmt);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
db.prepare('INSERT OR IGNORE INTO ran_migrations (name) VALUES (?)').run(
|
||||
migrationName,
|
||||
);
|
||||
console.log(`Applied migration: ${file}`);
|
||||
} catch (err) {
|
||||
console.error(`Failed to apply migration ${file}:`, err);
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
38
apps/storage-service/src/lib/db/schema.ts
Normal file
38
apps/storage-service/src/lib/db/schema.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import { sql } from 'drizzle-orm';
|
||||
import { text, integer, sqliteTable } from 'drizzle-orm/sqlite-core';
|
||||
import { Block } from '../types';
|
||||
import { SearchSources } from '../agents/search/types';
|
||||
|
||||
export const messages = sqliteTable('messages', {
|
||||
id: integer('id').primaryKey(),
|
||||
messageId: text('messageId').notNull(),
|
||||
chatId: text('chatId').notNull(),
|
||||
backendId: text('backendId').notNull(),
|
||||
query: text('query').notNull(),
|
||||
createdAt: text('createdAt').notNull(),
|
||||
responseBlocks: text('responseBlocks', { mode: 'json' })
|
||||
.$type<Block[]>()
|
||||
.default(sql`'[]'`),
|
||||
status: text({ enum: ['answering', 'completed', 'error'] }).default(
|
||||
'answering',
|
||||
),
|
||||
});
|
||||
|
||||
interface DBFile {
|
||||
name: string;
|
||||
fileId: string;
|
||||
}
|
||||
|
||||
export const chats = sqliteTable('chats', {
|
||||
id: text('id').primaryKey(),
|
||||
title: text('title').notNull(),
|
||||
createdAt: text('createdAt').notNull(),
|
||||
sources: text('sources', {
|
||||
mode: 'json',
|
||||
})
|
||||
.$type<SearchSources[]>()
|
||||
.default(sql`'[]'`),
|
||||
files: text('files', { mode: 'json' })
|
||||
.$type<DBFile[]>()
|
||||
.default(sql`'[]'`),
|
||||
});
|
||||
Reference in New Issue
Block a user