06fe57c765
Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
553 lines
15 KiB
Go
553 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/gooseek/backend/internal/computer"
|
|
"github.com/gooseek/backend/internal/computer/connectors"
|
|
"github.com/gooseek/backend/internal/db"
|
|
"github.com/gooseek/backend/internal/llm"
|
|
"github.com/gooseek/backend/pkg/config"
|
|
"github.com/gooseek/backend/pkg/middleware"
|
|
"github.com/gofiber/fiber/v2"
|
|
"github.com/gofiber/fiber/v2/middleware/cors"
|
|
"github.com/gofiber/fiber/v2/middleware/recover"
|
|
)
|
|
|
|
func main() {
|
|
cfg, err := config.Load()
|
|
if err != nil {
|
|
log.Fatalf("Failed to load config: %v", err)
|
|
}
|
|
|
|
var database *db.PostgresDB
|
|
maxRetries := 30
|
|
for i := 0; i < maxRetries; i++ {
|
|
database, err = db.NewPostgresDB(cfg.DatabaseURL)
|
|
if err == nil {
|
|
log.Println("PostgreSQL connected successfully")
|
|
break
|
|
}
|
|
log.Printf("Waiting for database (attempt %d/%d): %v", i+1, maxRetries, err)
|
|
time.Sleep(2 * time.Second)
|
|
}
|
|
if err != nil {
|
|
log.Fatalf("Failed to connect to database after %d attempts: %v", maxRetries, err)
|
|
}
|
|
|
|
taskRepo := db.NewComputerTaskRepo(database.DB())
|
|
memoryRepo := db.NewComputerMemoryRepo(database.DB())
|
|
artifactRepo := db.NewComputerArtifactRepo(database.DB())
|
|
|
|
if err := taskRepo.Migrate(); err != nil {
|
|
log.Printf("Task repo migration warning: %v", err)
|
|
}
|
|
if err := memoryRepo.Migrate(); err != nil {
|
|
log.Printf("Memory repo migration warning: %v", err)
|
|
}
|
|
if err := artifactRepo.Migrate(); err != nil {
|
|
log.Printf("Artifact repo migration warning: %v", err)
|
|
}
|
|
|
|
registry := llm.NewModelRegistry()
|
|
setupModels(registry, cfg)
|
|
|
|
connectorHub := connectors.NewConnectorHub()
|
|
setupConnectors(connectorHub, cfg)
|
|
|
|
comp := computer.NewComputer(computer.ComputerConfig{
|
|
MaxParallelTasks: 10,
|
|
MaxSubTasks: 20,
|
|
TaskTimeout: 30 * time.Minute,
|
|
SubTaskTimeout: 5 * time.Minute,
|
|
TotalBudget: 1.0,
|
|
EnableSandbox: true,
|
|
EnableScheduling: true,
|
|
SandboxImage: getEnv("SANDBOX_IMAGE", "gooseek/sandbox:latest"),
|
|
}, computer.Dependencies{
|
|
Registry: registry,
|
|
TaskRepo: taskRepo,
|
|
MemoryRepo: memoryRepo,
|
|
})
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
comp.StartScheduler(ctx)
|
|
|
|
app := fiber.New(fiber.Config{
|
|
ErrorHandler: func(c *fiber.Ctx, err error) error {
|
|
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
|
|
"error": err.Error(),
|
|
})
|
|
},
|
|
})
|
|
|
|
app.Use(recover.New())
|
|
app.Use(cors.New(cors.Config{
|
|
AllowOrigins: "*",
|
|
AllowHeaders: "Origin, Content-Type, Accept, Authorization",
|
|
AllowMethods: "GET, POST, PUT, DELETE, OPTIONS",
|
|
}))
|
|
app.Use(middleware.Logging(middleware.LoggingConfig{}))
|
|
|
|
app.Get("/health", func(c *fiber.Ctx) error {
|
|
return c.JSON(fiber.Map{
|
|
"status": "ok",
|
|
"service": "computer-svc",
|
|
"models": registry.Count(),
|
|
})
|
|
})
|
|
|
|
api := app.Group("/api/v1/computer")
|
|
|
|
api.Post("/execute", func(c *fiber.Ctx) error {
|
|
var req struct {
|
|
Query string `json:"query"`
|
|
UserID string `json:"userId"`
|
|
Options computer.ExecuteOptions `json:"options"`
|
|
}
|
|
|
|
if err := c.BodyParser(&req); err != nil {
|
|
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
|
|
}
|
|
|
|
if req.Query == "" {
|
|
return c.Status(400).JSON(fiber.Map{"error": "query is required"})
|
|
}
|
|
|
|
if req.UserID == "" || req.UserID == "anonymous" {
|
|
req.UserID = "00000000-0000-0000-0000-000000000000"
|
|
}
|
|
|
|
task, err := comp.Execute(c.Context(), req.UserID, req.Query, req.Options)
|
|
if err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
return c.JSON(task)
|
|
})
|
|
|
|
api.Get("/tasks", func(c *fiber.Ctx) error {
|
|
userID := c.Query("userId", "")
|
|
limit := c.QueryInt("limit", 20)
|
|
offset := c.QueryInt("offset", 0)
|
|
|
|
if userID == "" || userID == "anonymous" {
|
|
userID = "00000000-0000-0000-0000-000000000000"
|
|
}
|
|
|
|
tasks, err := comp.GetUserTasks(c.Context(), userID, limit, offset)
|
|
if err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
return c.JSON(fiber.Map{
|
|
"tasks": tasks,
|
|
"count": len(tasks),
|
|
})
|
|
})
|
|
|
|
api.Get("/tasks/:id", func(c *fiber.Ctx) error {
|
|
taskID := c.Params("id")
|
|
|
|
task, err := comp.GetStatus(c.Context(), taskID)
|
|
if err != nil {
|
|
return c.Status(404).JSON(fiber.Map{"error": "task not found"})
|
|
}
|
|
|
|
return c.JSON(task)
|
|
})
|
|
|
|
api.Get("/tasks/:id/stream", func(c *fiber.Ctx) error {
|
|
taskID := c.Params("id")
|
|
|
|
c.Set("Content-Type", "text/event-stream")
|
|
c.Set("Cache-Control", "no-cache")
|
|
c.Set("Connection", "keep-alive")
|
|
c.Set("Transfer-Encoding", "chunked")
|
|
|
|
eventCh, err := comp.Stream(c.Context(), taskID)
|
|
if err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
c.Context().SetBodyStreamWriter(func(w *bufio.Writer) {
|
|
for event := range eventCh {
|
|
data, _ := json.Marshal(event)
|
|
fmt.Fprintf(w, "data: %s\n\n", data)
|
|
w.Flush()
|
|
}
|
|
})
|
|
|
|
return nil
|
|
})
|
|
|
|
api.Post("/tasks/:id/resume", func(c *fiber.Ctx) error {
|
|
taskID := c.Params("id")
|
|
|
|
var req struct {
|
|
UserInput string `json:"userInput"`
|
|
}
|
|
|
|
if err := c.BodyParser(&req); err != nil {
|
|
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
|
|
}
|
|
|
|
if err := comp.Resume(c.Context(), taskID, req.UserInput); err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
return c.JSON(fiber.Map{"status": "resumed"})
|
|
})
|
|
|
|
api.Delete("/tasks/:id", func(c *fiber.Ctx) error {
|
|
taskID := c.Params("id")
|
|
|
|
if err := comp.Cancel(c.Context(), taskID); err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
return c.JSON(fiber.Map{"status": "cancelled"})
|
|
})
|
|
|
|
api.Get("/tasks/:id/artifacts", func(c *fiber.Ctx) error {
|
|
taskID := c.Params("id")
|
|
|
|
artifacts, err := artifactRepo.GetByTaskID(c.Context(), taskID)
|
|
if err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
return c.JSON(fiber.Map{
|
|
"artifacts": artifacts,
|
|
"count": len(artifacts),
|
|
})
|
|
})
|
|
|
|
api.Get("/artifacts/:id", func(c *fiber.Ctx) error {
|
|
artifactID := c.Params("id")
|
|
|
|
artifact, err := artifactRepo.GetByID(c.Context(), artifactID)
|
|
if err != nil {
|
|
return c.Status(404).JSON(fiber.Map{"error": "artifact not found"})
|
|
}
|
|
|
|
return c.JSON(artifact)
|
|
})
|
|
|
|
api.Get("/artifacts/:id/download", func(c *fiber.Ctx) error {
|
|
artifactID := c.Params("id")
|
|
|
|
artifact, err := artifactRepo.GetByID(c.Context(), artifactID)
|
|
if err != nil {
|
|
return c.Status(404).JSON(fiber.Map{"error": "artifact not found"})
|
|
}
|
|
|
|
if artifact.MimeType != "" {
|
|
c.Set("Content-Type", artifact.MimeType)
|
|
} else {
|
|
c.Set("Content-Type", "application/octet-stream")
|
|
}
|
|
c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", artifact.Name))
|
|
|
|
return c.Send(artifact.Content)
|
|
})
|
|
|
|
api.Get("/models", func(c *fiber.Ctx) error {
|
|
models := registry.GetAll()
|
|
return c.JSON(fiber.Map{
|
|
"models": models,
|
|
"count": len(models),
|
|
})
|
|
})
|
|
|
|
api.Get("/connectors", func(c *fiber.Ctx) error {
|
|
info := connectorHub.GetInfo()
|
|
return c.JSON(fiber.Map{
|
|
"connectors": info,
|
|
"count": len(info),
|
|
})
|
|
})
|
|
|
|
api.Post("/connectors/:id/execute", func(c *fiber.Ctx) error {
|
|
connectorID := c.Params("id")
|
|
|
|
var req struct {
|
|
Action string `json:"action"`
|
|
Params map[string]interface{} `json:"params"`
|
|
}
|
|
|
|
if err := c.BodyParser(&req); err != nil {
|
|
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
|
|
}
|
|
|
|
result, err := connectorHub.Execute(c.Context(), connectorID, req.Action, req.Params)
|
|
if err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
return c.JSON(result)
|
|
})
|
|
|
|
port := getEnv("COMPUTER_SVC_PORT", "3030")
|
|
addr := ":" + port
|
|
|
|
go func() {
|
|
log.Printf("Computer service starting on %s", addr)
|
|
if err := app.Listen(addr); err != nil {
|
|
log.Fatalf("Failed to start server: %v", err)
|
|
}
|
|
}()
|
|
|
|
quit := make(chan os.Signal, 1)
|
|
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
|
<-quit
|
|
|
|
log.Println("Shutting down...")
|
|
comp.StopScheduler()
|
|
app.Shutdown()
|
|
}
|
|
|
|
func setupModels(registry *llm.ModelRegistry, cfg *config.Config) {
|
|
// Timeweb Cloud AI (приоритетный провайдер для России)
|
|
if cfg.TimewebAgentAccessID != "" && cfg.TimewebAPIKey != "" {
|
|
timewebClient, err := llm.NewTimewebClient(llm.TimewebConfig{
|
|
ProviderID: "timeweb",
|
|
ModelKey: "gpt-4o",
|
|
BaseURL: cfg.TimewebAPIBaseURL,
|
|
AgentAccessID: cfg.TimewebAgentAccessID,
|
|
APIKey: cfg.TimewebAPIKey,
|
|
ProxySource: cfg.TimewebProxySource,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "timeweb-gpt-4o",
|
|
Provider: "timeweb",
|
|
Model: "gpt-4o",
|
|
Capabilities: []llm.ModelCapability{llm.CapSearch, llm.CapFast, llm.CapVision, llm.CapCoding, llm.CapCreative, llm.CapReasoning},
|
|
CostPer1K: 0.005,
|
|
MaxContext: 128000,
|
|
MaxTokens: 16384,
|
|
Priority: 0,
|
|
Description: "GPT-4o via Timeweb Cloud AI",
|
|
}, timewebClient)
|
|
log.Println("Timeweb GPT-4o registered")
|
|
} else {
|
|
log.Printf("Failed to create Timeweb client: %v", err)
|
|
}
|
|
|
|
timewebMiniClient, err := llm.NewTimewebClient(llm.TimewebConfig{
|
|
ProviderID: "timeweb",
|
|
ModelKey: "gpt-4o-mini",
|
|
BaseURL: cfg.TimewebAPIBaseURL,
|
|
AgentAccessID: cfg.TimewebAgentAccessID,
|
|
APIKey: cfg.TimewebAPIKey,
|
|
ProxySource: cfg.TimewebProxySource,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "timeweb-gpt-4o-mini",
|
|
Provider: "timeweb",
|
|
Model: "gpt-4o-mini",
|
|
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapCoding},
|
|
CostPer1K: 0.00015,
|
|
MaxContext: 128000,
|
|
MaxTokens: 16384,
|
|
Priority: 0,
|
|
Description: "GPT-4o-mini via Timeweb Cloud AI",
|
|
}, timewebMiniClient)
|
|
log.Println("Timeweb GPT-4o-mini registered")
|
|
}
|
|
}
|
|
|
|
// OpenAI прямой (fallback если Timeweb недоступен)
|
|
if cfg.OpenAIAPIKey != "" {
|
|
openaiClient, err := llm.NewOpenAIClient(llm.ProviderConfig{
|
|
ProviderID: "openai",
|
|
ModelKey: "gpt-4o",
|
|
APIKey: cfg.OpenAIAPIKey,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "gpt-4o",
|
|
Provider: "openai",
|
|
Model: "gpt-4o",
|
|
Capabilities: []llm.ModelCapability{llm.CapSearch, llm.CapFast, llm.CapVision, llm.CapCoding, llm.CapCreative},
|
|
CostPer1K: 0.005,
|
|
MaxContext: 128000,
|
|
MaxTokens: 16384,
|
|
Priority: 10,
|
|
}, openaiClient)
|
|
}
|
|
|
|
miniClient, err := llm.NewOpenAIClient(llm.ProviderConfig{
|
|
ProviderID: "openai",
|
|
ModelKey: "gpt-4o-mini",
|
|
APIKey: cfg.OpenAIAPIKey,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "gpt-4o-mini",
|
|
Provider: "openai",
|
|
Model: "gpt-4o-mini",
|
|
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapCoding},
|
|
CostPer1K: 0.00015,
|
|
MaxContext: 128000,
|
|
MaxTokens: 16384,
|
|
Priority: 10,
|
|
}, miniClient)
|
|
}
|
|
}
|
|
|
|
if cfg.AnthropicAPIKey != "" {
|
|
opusClient, err := llm.NewAnthropicClient(llm.ProviderConfig{
|
|
ProviderID: "anthropic",
|
|
ModelKey: "claude-3-opus-20240229",
|
|
APIKey: cfg.AnthropicAPIKey,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "claude-3-opus",
|
|
Provider: "anthropic",
|
|
Model: "claude-3-opus-20240229",
|
|
Capabilities: []llm.ModelCapability{llm.CapReasoning, llm.CapCoding, llm.CapCreative, llm.CapLongContext},
|
|
CostPer1K: 0.015,
|
|
MaxContext: 200000,
|
|
MaxTokens: 4096,
|
|
Priority: 1,
|
|
}, opusClient)
|
|
}
|
|
|
|
sonnetClient, err := llm.NewAnthropicClient(llm.ProviderConfig{
|
|
ProviderID: "anthropic",
|
|
ModelKey: "claude-3-5-sonnet-20241022",
|
|
APIKey: cfg.AnthropicAPIKey,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "claude-3-sonnet",
|
|
Provider: "anthropic",
|
|
Model: "claude-3-5-sonnet-20241022",
|
|
Capabilities: []llm.ModelCapability{llm.CapCoding, llm.CapCreative, llm.CapFast},
|
|
CostPer1K: 0.003,
|
|
MaxContext: 200000,
|
|
MaxTokens: 8192,
|
|
Priority: 1,
|
|
}, sonnetClient)
|
|
}
|
|
}
|
|
|
|
if cfg.GeminiAPIKey != "" {
|
|
geminiClient, err := llm.NewGeminiClient(llm.ProviderConfig{
|
|
ProviderID: "gemini",
|
|
ModelKey: "gemini-1.5-pro",
|
|
APIKey: cfg.GeminiAPIKey,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "gemini-1.5-pro",
|
|
Provider: "gemini",
|
|
Model: "gemini-1.5-pro",
|
|
Capabilities: []llm.ModelCapability{llm.CapLongContext, llm.CapSearch, llm.CapVision, llm.CapMath},
|
|
CostPer1K: 0.00125,
|
|
MaxContext: 2000000,
|
|
MaxTokens: 8192,
|
|
Priority: 1,
|
|
}, geminiClient)
|
|
}
|
|
|
|
flashClient, err := llm.NewGeminiClient(llm.ProviderConfig{
|
|
ProviderID: "gemini",
|
|
ModelKey: "gemini-1.5-flash",
|
|
APIKey: cfg.GeminiAPIKey,
|
|
})
|
|
if err == nil {
|
|
registry.Register(llm.ModelSpec{
|
|
ID: "gemini-1.5-flash",
|
|
Provider: "gemini",
|
|
Model: "gemini-1.5-flash",
|
|
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapVision},
|
|
CostPer1K: 0.000075,
|
|
MaxContext: 1000000,
|
|
MaxTokens: 8192,
|
|
Priority: 2,
|
|
}, flashClient)
|
|
}
|
|
}
|
|
|
|
log.Printf("Registered %d models", registry.Count())
|
|
}
|
|
|
|
func setupConnectors(hub *connectors.ConnectorHub, cfg *config.Config) {
|
|
if smtpHost := getEnv("SMTP_HOST", ""); smtpHost != "" {
|
|
emailConn := connectors.NewEmailConnector(connectors.EmailConfig{
|
|
SMTPHost: smtpHost,
|
|
SMTPPort: getEnvInt("SMTP_PORT", 587),
|
|
Username: getEnv("SMTP_USERNAME", ""),
|
|
Password: getEnv("SMTP_PASSWORD", ""),
|
|
FromAddress: getEnv("SMTP_FROM", ""),
|
|
FromName: getEnv("SMTP_FROM_NAME", "GooSeek Computer"),
|
|
UseTLS: true,
|
|
AllowHTML: true,
|
|
})
|
|
hub.Register(emailConn)
|
|
log.Println("Email connector registered")
|
|
}
|
|
|
|
if botToken := getEnv("TELEGRAM_BOT_TOKEN", ""); botToken != "" {
|
|
tgConn := connectors.NewTelegramConnector(connectors.TelegramConfig{
|
|
BotToken: botToken,
|
|
})
|
|
hub.Register(tgConn)
|
|
log.Println("Telegram connector registered")
|
|
}
|
|
|
|
webhookConn := connectors.NewWebhookConnector(connectors.WebhookConfig{
|
|
Timeout: 30 * time.Second,
|
|
MaxRetries: 3,
|
|
})
|
|
hub.Register(webhookConn)
|
|
log.Println("Webhook connector registered")
|
|
|
|
if s3Endpoint := getEnv("S3_ENDPOINT", ""); s3Endpoint != "" {
|
|
storageConn, err := connectors.NewStorageConnector(connectors.StorageConfig{
|
|
Endpoint: s3Endpoint,
|
|
AccessKeyID: getEnv("S3_ACCESS_KEY", ""),
|
|
SecretAccessKey: getEnv("S3_SECRET_KEY", ""),
|
|
BucketName: getEnv("S3_BUCKET", "gooseek-artifacts"),
|
|
UseSSL: getEnv("S3_USE_SSL", "true") == "true",
|
|
Region: getEnv("S3_REGION", "us-east-1"),
|
|
PublicURL: getEnv("S3_PUBLIC_URL", ""),
|
|
})
|
|
if err == nil {
|
|
hub.Register(storageConn)
|
|
log.Println("Storage connector registered")
|
|
}
|
|
}
|
|
}
|
|
|
|
func getEnv(key, defaultValue string) string {
|
|
if value := os.Getenv(key); value != "" {
|
|
return value
|
|
}
|
|
return defaultValue
|
|
}
|
|
|
|
func getEnvInt(key string, defaultValue int) int {
|
|
if value := os.Getenv(key); value != "" {
|
|
var i int
|
|
fmt.Sscanf(value, "%d", &i)
|
|
return i
|
|
}
|
|
return defaultValue
|
|
}
|