feat: Go backend, enhanced search, new widgets, Docker deploy

Major changes:
- Add Go backend (backend/) with microservices architecture
- Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler,
  proxy-manager, media-search, fastClassifier, language detection
- New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard,
  UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions
- Improved discover-svc with discover-db integration
- Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md)
- Library-svc: project_id schema migration
- Remove deprecated finance-svc and travel-svc
- Localization improvements across services

Made-with: Cursor
This commit is contained in:
home
2026-02-27 04:15:32 +03:00
parent 328d968f3f
commit 06fe57c765
285 changed files with 53132 additions and 1871 deletions

View File

@@ -0,0 +1,552 @@
package main
import (
"bufio"
"context"
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"time"
"github.com/gooseek/backend/internal/computer"
"github.com/gooseek/backend/internal/computer/connectors"
"github.com/gooseek/backend/internal/db"
"github.com/gooseek/backend/internal/llm"
"github.com/gooseek/backend/pkg/config"
"github.com/gooseek/backend/pkg/middleware"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
"github.com/gofiber/fiber/v2/middleware/recover"
)
func main() {
cfg, err := config.Load()
if err != nil {
log.Fatalf("Failed to load config: %v", err)
}
var database *db.PostgresDB
maxRetries := 30
for i := 0; i < maxRetries; i++ {
database, err = db.NewPostgresDB(cfg.DatabaseURL)
if err == nil {
log.Println("PostgreSQL connected successfully")
break
}
log.Printf("Waiting for database (attempt %d/%d): %v", i+1, maxRetries, err)
time.Sleep(2 * time.Second)
}
if err != nil {
log.Fatalf("Failed to connect to database after %d attempts: %v", maxRetries, err)
}
taskRepo := db.NewComputerTaskRepo(database.DB())
memoryRepo := db.NewComputerMemoryRepo(database.DB())
artifactRepo := db.NewComputerArtifactRepo(database.DB())
if err := taskRepo.Migrate(); err != nil {
log.Printf("Task repo migration warning: %v", err)
}
if err := memoryRepo.Migrate(); err != nil {
log.Printf("Memory repo migration warning: %v", err)
}
if err := artifactRepo.Migrate(); err != nil {
log.Printf("Artifact repo migration warning: %v", err)
}
registry := llm.NewModelRegistry()
setupModels(registry, cfg)
connectorHub := connectors.NewConnectorHub()
setupConnectors(connectorHub, cfg)
comp := computer.NewComputer(computer.ComputerConfig{
MaxParallelTasks: 10,
MaxSubTasks: 20,
TaskTimeout: 30 * time.Minute,
SubTaskTimeout: 5 * time.Minute,
TotalBudget: 1.0,
EnableSandbox: true,
EnableScheduling: true,
SandboxImage: getEnv("SANDBOX_IMAGE", "gooseek/sandbox:latest"),
}, computer.Dependencies{
Registry: registry,
TaskRepo: taskRepo,
MemoryRepo: memoryRepo,
})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
comp.StartScheduler(ctx)
app := fiber.New(fiber.Config{
ErrorHandler: func(c *fiber.Ctx, err error) error {
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
"error": err.Error(),
})
},
})
app.Use(recover.New())
app.Use(cors.New(cors.Config{
AllowOrigins: "*",
AllowHeaders: "Origin, Content-Type, Accept, Authorization",
AllowMethods: "GET, POST, PUT, DELETE, OPTIONS",
}))
app.Use(middleware.Logging(middleware.LoggingConfig{}))
app.Get("/health", func(c *fiber.Ctx) error {
return c.JSON(fiber.Map{
"status": "ok",
"service": "computer-svc",
"models": registry.Count(),
})
})
api := app.Group("/api/v1/computer")
api.Post("/execute", func(c *fiber.Ctx) error {
var req struct {
Query string `json:"query"`
UserID string `json:"userId"`
Options computer.ExecuteOptions `json:"options"`
}
if err := c.BodyParser(&req); err != nil {
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
}
if req.Query == "" {
return c.Status(400).JSON(fiber.Map{"error": "query is required"})
}
if req.UserID == "" || req.UserID == "anonymous" {
req.UserID = "00000000-0000-0000-0000-000000000000"
}
task, err := comp.Execute(c.Context(), req.UserID, req.Query, req.Options)
if err != nil {
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
}
return c.JSON(task)
})
api.Get("/tasks", func(c *fiber.Ctx) error {
userID := c.Query("userId", "")
limit := c.QueryInt("limit", 20)
offset := c.QueryInt("offset", 0)
if userID == "" || userID == "anonymous" {
userID = "00000000-0000-0000-0000-000000000000"
}
tasks, err := comp.GetUserTasks(c.Context(), userID, limit, offset)
if err != nil {
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
}
return c.JSON(fiber.Map{
"tasks": tasks,
"count": len(tasks),
})
})
api.Get("/tasks/:id", func(c *fiber.Ctx) error {
taskID := c.Params("id")
task, err := comp.GetStatus(c.Context(), taskID)
if err != nil {
return c.Status(404).JSON(fiber.Map{"error": "task not found"})
}
return c.JSON(task)
})
api.Get("/tasks/:id/stream", func(c *fiber.Ctx) error {
taskID := c.Params("id")
c.Set("Content-Type", "text/event-stream")
c.Set("Cache-Control", "no-cache")
c.Set("Connection", "keep-alive")
c.Set("Transfer-Encoding", "chunked")
eventCh, err := comp.Stream(c.Context(), taskID)
if err != nil {
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
}
c.Context().SetBodyStreamWriter(func(w *bufio.Writer) {
for event := range eventCh {
data, _ := json.Marshal(event)
fmt.Fprintf(w, "data: %s\n\n", data)
w.Flush()
}
})
return nil
})
api.Post("/tasks/:id/resume", func(c *fiber.Ctx) error {
taskID := c.Params("id")
var req struct {
UserInput string `json:"userInput"`
}
if err := c.BodyParser(&req); err != nil {
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
}
if err := comp.Resume(c.Context(), taskID, req.UserInput); err != nil {
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
}
return c.JSON(fiber.Map{"status": "resumed"})
})
api.Delete("/tasks/:id", func(c *fiber.Ctx) error {
taskID := c.Params("id")
if err := comp.Cancel(c.Context(), taskID); err != nil {
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
}
return c.JSON(fiber.Map{"status": "cancelled"})
})
api.Get("/tasks/:id/artifacts", func(c *fiber.Ctx) error {
taskID := c.Params("id")
artifacts, err := artifactRepo.GetByTaskID(c.Context(), taskID)
if err != nil {
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
}
return c.JSON(fiber.Map{
"artifacts": artifacts,
"count": len(artifacts),
})
})
api.Get("/artifacts/:id", func(c *fiber.Ctx) error {
artifactID := c.Params("id")
artifact, err := artifactRepo.GetByID(c.Context(), artifactID)
if err != nil {
return c.Status(404).JSON(fiber.Map{"error": "artifact not found"})
}
return c.JSON(artifact)
})
api.Get("/artifacts/:id/download", func(c *fiber.Ctx) error {
artifactID := c.Params("id")
artifact, err := artifactRepo.GetByID(c.Context(), artifactID)
if err != nil {
return c.Status(404).JSON(fiber.Map{"error": "artifact not found"})
}
if artifact.MimeType != "" {
c.Set("Content-Type", artifact.MimeType)
} else {
c.Set("Content-Type", "application/octet-stream")
}
c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", artifact.Name))
return c.Send(artifact.Content)
})
api.Get("/models", func(c *fiber.Ctx) error {
models := registry.GetAll()
return c.JSON(fiber.Map{
"models": models,
"count": len(models),
})
})
api.Get("/connectors", func(c *fiber.Ctx) error {
info := connectorHub.GetInfo()
return c.JSON(fiber.Map{
"connectors": info,
"count": len(info),
})
})
api.Post("/connectors/:id/execute", func(c *fiber.Ctx) error {
connectorID := c.Params("id")
var req struct {
Action string `json:"action"`
Params map[string]interface{} `json:"params"`
}
if err := c.BodyParser(&req); err != nil {
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
}
result, err := connectorHub.Execute(c.Context(), connectorID, req.Action, req.Params)
if err != nil {
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
}
return c.JSON(result)
})
port := getEnv("COMPUTER_SVC_PORT", "3030")
addr := ":" + port
go func() {
log.Printf("Computer service starting on %s", addr)
if err := app.Listen(addr); err != nil {
log.Fatalf("Failed to start server: %v", err)
}
}()
quit := make(chan os.Signal, 1)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
<-quit
log.Println("Shutting down...")
comp.StopScheduler()
app.Shutdown()
}
func setupModels(registry *llm.ModelRegistry, cfg *config.Config) {
// Timeweb Cloud AI (приоритетный провайдер для России)
if cfg.TimewebAgentAccessID != "" && cfg.TimewebAPIKey != "" {
timewebClient, err := llm.NewTimewebClient(llm.TimewebConfig{
ProviderID: "timeweb",
ModelKey: "gpt-4o",
BaseURL: cfg.TimewebAPIBaseURL,
AgentAccessID: cfg.TimewebAgentAccessID,
APIKey: cfg.TimewebAPIKey,
ProxySource: cfg.TimewebProxySource,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "timeweb-gpt-4o",
Provider: "timeweb",
Model: "gpt-4o",
Capabilities: []llm.ModelCapability{llm.CapSearch, llm.CapFast, llm.CapVision, llm.CapCoding, llm.CapCreative, llm.CapReasoning},
CostPer1K: 0.005,
MaxContext: 128000,
MaxTokens: 16384,
Priority: 0,
Description: "GPT-4o via Timeweb Cloud AI",
}, timewebClient)
log.Println("Timeweb GPT-4o registered")
} else {
log.Printf("Failed to create Timeweb client: %v", err)
}
timewebMiniClient, err := llm.NewTimewebClient(llm.TimewebConfig{
ProviderID: "timeweb",
ModelKey: "gpt-4o-mini",
BaseURL: cfg.TimewebAPIBaseURL,
AgentAccessID: cfg.TimewebAgentAccessID,
APIKey: cfg.TimewebAPIKey,
ProxySource: cfg.TimewebProxySource,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "timeweb-gpt-4o-mini",
Provider: "timeweb",
Model: "gpt-4o-mini",
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapCoding},
CostPer1K: 0.00015,
MaxContext: 128000,
MaxTokens: 16384,
Priority: 0,
Description: "GPT-4o-mini via Timeweb Cloud AI",
}, timewebMiniClient)
log.Println("Timeweb GPT-4o-mini registered")
}
}
// OpenAI прямой (fallback если Timeweb недоступен)
if cfg.OpenAIAPIKey != "" {
openaiClient, err := llm.NewOpenAIClient(llm.ProviderConfig{
ProviderID: "openai",
ModelKey: "gpt-4o",
APIKey: cfg.OpenAIAPIKey,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "gpt-4o",
Provider: "openai",
Model: "gpt-4o",
Capabilities: []llm.ModelCapability{llm.CapSearch, llm.CapFast, llm.CapVision, llm.CapCoding, llm.CapCreative},
CostPer1K: 0.005,
MaxContext: 128000,
MaxTokens: 16384,
Priority: 10,
}, openaiClient)
}
miniClient, err := llm.NewOpenAIClient(llm.ProviderConfig{
ProviderID: "openai",
ModelKey: "gpt-4o-mini",
APIKey: cfg.OpenAIAPIKey,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "gpt-4o-mini",
Provider: "openai",
Model: "gpt-4o-mini",
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapCoding},
CostPer1K: 0.00015,
MaxContext: 128000,
MaxTokens: 16384,
Priority: 10,
}, miniClient)
}
}
if cfg.AnthropicAPIKey != "" {
opusClient, err := llm.NewAnthropicClient(llm.ProviderConfig{
ProviderID: "anthropic",
ModelKey: "claude-3-opus-20240229",
APIKey: cfg.AnthropicAPIKey,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "claude-3-opus",
Provider: "anthropic",
Model: "claude-3-opus-20240229",
Capabilities: []llm.ModelCapability{llm.CapReasoning, llm.CapCoding, llm.CapCreative, llm.CapLongContext},
CostPer1K: 0.015,
MaxContext: 200000,
MaxTokens: 4096,
Priority: 1,
}, opusClient)
}
sonnetClient, err := llm.NewAnthropicClient(llm.ProviderConfig{
ProviderID: "anthropic",
ModelKey: "claude-3-5-sonnet-20241022",
APIKey: cfg.AnthropicAPIKey,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "claude-3-sonnet",
Provider: "anthropic",
Model: "claude-3-5-sonnet-20241022",
Capabilities: []llm.ModelCapability{llm.CapCoding, llm.CapCreative, llm.CapFast},
CostPer1K: 0.003,
MaxContext: 200000,
MaxTokens: 8192,
Priority: 1,
}, sonnetClient)
}
}
if cfg.GeminiAPIKey != "" {
geminiClient, err := llm.NewGeminiClient(llm.ProviderConfig{
ProviderID: "gemini",
ModelKey: "gemini-1.5-pro",
APIKey: cfg.GeminiAPIKey,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "gemini-1.5-pro",
Provider: "gemini",
Model: "gemini-1.5-pro",
Capabilities: []llm.ModelCapability{llm.CapLongContext, llm.CapSearch, llm.CapVision, llm.CapMath},
CostPer1K: 0.00125,
MaxContext: 2000000,
MaxTokens: 8192,
Priority: 1,
}, geminiClient)
}
flashClient, err := llm.NewGeminiClient(llm.ProviderConfig{
ProviderID: "gemini",
ModelKey: "gemini-1.5-flash",
APIKey: cfg.GeminiAPIKey,
})
if err == nil {
registry.Register(llm.ModelSpec{
ID: "gemini-1.5-flash",
Provider: "gemini",
Model: "gemini-1.5-flash",
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapVision},
CostPer1K: 0.000075,
MaxContext: 1000000,
MaxTokens: 8192,
Priority: 2,
}, flashClient)
}
}
log.Printf("Registered %d models", registry.Count())
}
func setupConnectors(hub *connectors.ConnectorHub, cfg *config.Config) {
if smtpHost := getEnv("SMTP_HOST", ""); smtpHost != "" {
emailConn := connectors.NewEmailConnector(connectors.EmailConfig{
SMTPHost: smtpHost,
SMTPPort: getEnvInt("SMTP_PORT", 587),
Username: getEnv("SMTP_USERNAME", ""),
Password: getEnv("SMTP_PASSWORD", ""),
FromAddress: getEnv("SMTP_FROM", ""),
FromName: getEnv("SMTP_FROM_NAME", "GooSeek Computer"),
UseTLS: true,
AllowHTML: true,
})
hub.Register(emailConn)
log.Println("Email connector registered")
}
if botToken := getEnv("TELEGRAM_BOT_TOKEN", ""); botToken != "" {
tgConn := connectors.NewTelegramConnector(connectors.TelegramConfig{
BotToken: botToken,
})
hub.Register(tgConn)
log.Println("Telegram connector registered")
}
webhookConn := connectors.NewWebhookConnector(connectors.WebhookConfig{
Timeout: 30 * time.Second,
MaxRetries: 3,
})
hub.Register(webhookConn)
log.Println("Webhook connector registered")
if s3Endpoint := getEnv("S3_ENDPOINT", ""); s3Endpoint != "" {
storageConn, err := connectors.NewStorageConnector(connectors.StorageConfig{
Endpoint: s3Endpoint,
AccessKeyID: getEnv("S3_ACCESS_KEY", ""),
SecretAccessKey: getEnv("S3_SECRET_KEY", ""),
BucketName: getEnv("S3_BUCKET", "gooseek-artifacts"),
UseSSL: getEnv("S3_USE_SSL", "true") == "true",
Region: getEnv("S3_REGION", "us-east-1"),
PublicURL: getEnv("S3_PUBLIC_URL", ""),
})
if err == nil {
hub.Register(storageConn)
log.Println("Storage connector registered")
}
}
}
func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
func getEnvInt(key string, defaultValue int) int {
if value := os.Getenv(key); value != "" {
var i int
fmt.Sscanf(value, "%d", &i)
return i
}
return defaultValue
}