feat: Go backend, enhanced search, new widgets, Docker deploy

Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
2026-02-27 04:15:32 +03:00
parent 328d968f3f
commit 06fe57c765
285 changed files with 53132 additions and 1871 deletions
--- a/backend/cmd/thread-svc/main.go
+++ b/backend/cmd/thread-svc/main.go
@@ -0,0 +1,624 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/cors"
+	"github.com/gofiber/fiber/v2/middleware/logger"
+	"github.com/gooseek/backend/internal/db"
+	"github.com/gooseek/backend/internal/llm"
+	"github.com/gooseek/backend/internal/pages"
+	"github.com/gooseek/backend/pkg/config"
+	"github.com/gooseek/backend/pkg/middleware"
+)
+
+func main() {
+	cfg, err := config.Load()
+	if err != nil {
+		log.Fatal("Failed to load config:", err)
+	}
+
+	var database *db.PostgresDB
+	var threadRepo *db.ThreadRepository
+	var spaceRepo *db.SpaceRepository
+	var memoryRepo *db.MemoryRepository
+	var pageRepo *db.PageRepository
+
+	if cfg.DatabaseURL != "" {
+		maxRetries := 30
+		for i := 0; i < maxRetries; i++ {
+			database, err = db.NewPostgresDB(cfg.DatabaseURL)
+			if err == nil {
+				break
+			}
+			log.Printf("Waiting for database (attempt %d/%d): %v", i+1, maxRetries, err)
+			time.Sleep(2 * time.Second)
+		}
+		if err != nil {
+			log.Fatal("Database required for thread-svc:", err)
+		}
+		log.Println("PostgreSQL connected successfully")
+		defer database.Close()
+
+		ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+		defer cancel()
+
+		if err := database.RunMigrations(ctx); err != nil {
+			log.Printf("Base migrations warning: %v", err)
+		}
+
+		spaceRepo = db.NewSpaceRepository(database)
+		if err := spaceRepo.RunMigrations(ctx); err != nil {
+			log.Printf("Space migrations warning: %v", err)
+		}
+
+		threadRepo = db.NewThreadRepository(database)
+		if err := threadRepo.RunMigrations(ctx); err != nil {
+			log.Printf("Thread migrations warning: %v", err)
+		}
+
+		memoryRepo = db.NewMemoryRepository(database)
+		if err := memoryRepo.RunMigrations(ctx); err != nil {
+			log.Printf("Memory migrations warning: %v", err)
+		}
+
+		pageRepo = db.NewPageRepository(database)
+		if err := pageRepo.RunMigrations(ctx); err != nil {
+			log.Printf("Page migrations warning: %v", err)
+		}
+
+		log.Println("PostgreSQL connected, all migrations complete")
+	} else {
+		log.Fatal("DATABASE_URL required for thread-svc")
+	}
+
+	var llmClient llm.Client
+	if cfg.OpenAIAPIKey != "" {
+		llmClient, err = llm.NewClient(llm.ProviderConfig{
+			ProviderID: "openai",
+			ModelKey:   "gpt-4o-mini",
+			APIKey:     cfg.OpenAIAPIKey,
+		})
+		if err != nil {
+			log.Printf("Failed to create LLM client: %v", err)
+		}
+	}
+
+	app := fiber.New(fiber.Config{
+		BodyLimit:    10 * 1024 * 1024,
+		ReadTimeout:  30 * time.Second,
+		WriteTimeout: 30 * time.Second,
+	})
+
+	app.Use(logger.New())
+	app.Use(cors.New())
+
+	app.Get("/health", func(c *fiber.Ctx) error {
+		return c.JSON(fiber.Map{"status": "ok"})
+	})
+
+	app.Get("/ready", func(c *fiber.Ctx) error {
+		return c.JSON(fiber.Map{"status": "ready"})
+	})
+
+	threads := app.Group("/api/v1/threads", middleware.JWT(middleware.JWTConfig{
+		Secret:     cfg.JWTSecret,
+		AuthSvcURL: cfg.AuthSvcURL,
+		AllowGuest: true,
+	}))
+
+	threads.Get("/", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+		if userID == "" {
+			return c.JSON(fiber.Map{"threads": []interface{}{}})
+		}
+
+		limit := c.QueryInt("limit", 50)
+		offset := c.QueryInt("offset", 0)
+
+		threadList, err := threadRepo.GetByUserID(c.Context(), userID, limit, offset)
+		if err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to get threads"})
+		}
+
+		return c.JSON(fiber.Map{"threads": threadList})
+	})
+
+	threads.Post("/", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+		if userID == "" {
+			return c.Status(401).JSON(fiber.Map{"error": "Authentication required"})
+		}
+
+		var req struct {
+			Title     string  `json:"title"`
+			FocusMode string  `json:"focusMode"`
+			SpaceID   *string `json:"spaceId"`
+		}
+
+		if err := c.BodyParser(&req); err != nil {
+			return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
+		}
+
+		thread := &db.Thread{
+			UserID:    userID,
+			SpaceID:   req.SpaceID,
+			Title:     req.Title,
+			FocusMode: req.FocusMode,
+		}
+
+		if thread.Title == "" {
+			thread.Title = "New Thread"
+		}
+		if thread.FocusMode == "" {
+			thread.FocusMode = "all"
+		}
+
+		if err := threadRepo.Create(c.Context(), thread); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to create thread"})
+		}
+
+		return c.Status(201).JSON(thread)
+	})
+
+	threads.Get("/:id", func(c *fiber.Ctx) error {
+		threadID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		thread, err := threadRepo.GetByID(c.Context(), threadID)
+		if err != nil || thread == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
+		}
+
+		if thread.UserID != userID && !thread.IsPublic {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		messages, _ := threadRepo.GetMessages(c.Context(), threadID, 100, 0)
+		thread.Messages = messages
+
+		return c.JSON(thread)
+	})
+
+	threads.Post("/:id/messages", func(c *fiber.Ctx) error {
+		threadID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		thread, err := threadRepo.GetByID(c.Context(), threadID)
+		if err != nil || thread == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
+		}
+
+		if thread.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		var req struct {
+			Role             string                   `json:"role"`
+			Content          string                   `json:"content"`
+			Sources          []db.ThreadSource        `json:"sources"`
+			Widgets          []map[string]interface{} `json:"widgets"`
+			RelatedQuestions []string                 `json:"relatedQuestions"`
+			Model            string                   `json:"model"`
+			TokensUsed       int                      `json:"tokensUsed"`
+		}
+
+		if err := c.BodyParser(&req); err != nil {
+			return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
+		}
+
+		msg := &db.ThreadMessage{
+			ThreadID:         threadID,
+			Role:             req.Role,
+			Content:          req.Content,
+			Sources:          req.Sources,
+			Widgets:          req.Widgets,
+			RelatedQuestions: req.RelatedQuestions,
+			Model:            req.Model,
+			TokensUsed:       req.TokensUsed,
+		}
+
+		if err := threadRepo.AddMessage(c.Context(), msg); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to add message"})
+		}
+
+		if thread.Title == "New Thread" && req.Role == "user" {
+			threadRepo.GenerateTitle(c.Context(), threadID, req.Content)
+		}
+
+		return c.Status(201).JSON(msg)
+	})
+
+	threads.Post("/:id/share", func(c *fiber.Ctx) error {
+		threadID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		thread, err := threadRepo.GetByID(c.Context(), threadID)
+		if err != nil || thread == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
+		}
+
+		if thread.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		shareID := generateShareID()
+		if err := threadRepo.SetShareID(c.Context(), threadID, shareID); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to share thread"})
+		}
+
+		return c.JSON(fiber.Map{
+			"shareId":  shareID,
+			"shareUrl": fmt.Sprintf("/share/%s", shareID),
+		})
+	})
+
+	threads.Delete("/:id", func(c *fiber.Ctx) error {
+		threadID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		thread, err := threadRepo.GetByID(c.Context(), threadID)
+		if err != nil || thread == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
+		}
+
+		if thread.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		if err := threadRepo.Delete(c.Context(), threadID); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to delete thread"})
+		}
+
+		return c.Status(204).Send(nil)
+	})
+
+	share := app.Group("/api/v1/share")
+
+	share.Get("/:shareId", func(c *fiber.Ctx) error {
+		shareID := c.Params("shareId")
+
+		thread, err := threadRepo.GetByShareID(c.Context(), shareID)
+		if err != nil || thread == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Shared thread not found"})
+		}
+
+		messages, _ := threadRepo.GetMessages(c.Context(), thread.ID, 100, 0)
+		thread.Messages = messages
+
+		return c.JSON(thread)
+	})
+
+	spaces := app.Group("/api/v1/spaces", middleware.JWT(middleware.JWTConfig{
+		Secret:     cfg.JWTSecret,
+		AuthSvcURL: cfg.AuthSvcURL,
+		AllowGuest: false,
+	}))
+
+	spaces.Get("/", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+
+		spaceList, err := spaceRepo.GetByUserID(c.Context(), userID)
+		if err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to get spaces"})
+		}
+
+		return c.JSON(fiber.Map{"spaces": spaceList})
+	})
+
+	spaces.Post("/", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+
+		var req db.Space
+		if err := c.BodyParser(&req); err != nil {
+			return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
+		}
+
+		req.UserID = userID
+		if req.Name == "" {
+			return c.Status(400).JSON(fiber.Map{"error": "Name required"})
+		}
+
+		if err := spaceRepo.Create(c.Context(), &req); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to create space"})
+		}
+
+		return c.Status(201).JSON(req)
+	})
+
+	spaces.Get("/:id", func(c *fiber.Ctx) error {
+		spaceID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		space, err := spaceRepo.GetByID(c.Context(), spaceID)
+		if err != nil || space == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Space not found"})
+		}
+
+		if space.UserID != userID && !space.IsPublic {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		return c.JSON(space)
+	})
+
+	spaces.Put("/:id", func(c *fiber.Ctx) error {
+		spaceID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		space, err := spaceRepo.GetByID(c.Context(), spaceID)
+		if err != nil || space == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Space not found"})
+		}
+
+		if space.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		var req db.Space
+		if err := c.BodyParser(&req); err != nil {
+			return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
+		}
+
+		req.ID = spaceID
+		req.UserID = userID
+
+		if err := spaceRepo.Update(c.Context(), &req); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to update space"})
+		}
+
+		return c.JSON(req)
+	})
+
+	spaces.Delete("/:id", func(c *fiber.Ctx) error {
+		spaceID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		space, err := spaceRepo.GetByID(c.Context(), spaceID)
+		if err != nil || space == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Space not found"})
+		}
+
+		if space.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		if err := spaceRepo.Delete(c.Context(), spaceID); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to delete space"})
+		}
+
+		return c.Status(204).Send(nil)
+	})
+
+	memory := app.Group("/api/v1/memory", middleware.JWT(middleware.JWTConfig{
+		Secret:     cfg.JWTSecret,
+		AuthSvcURL: cfg.AuthSvcURL,
+		AllowGuest: false,
+	}))
+
+	memory.Get("/", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+		memType := c.Query("type", "")
+
+		memories, err := memoryRepo.GetByUserID(c.Context(), userID, memType, 50)
+		if err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to get memories"})
+		}
+
+		return c.JSON(fiber.Map{"memories": memories})
+	})
+
+	memory.Post("/", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+
+		var req db.UserMemory
+		if err := c.BodyParser(&req); err != nil {
+			return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
+		}
+
+		req.UserID = userID
+
+		if err := memoryRepo.Save(c.Context(), &req); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to save memory"})
+		}
+
+		return c.Status(201).JSON(req)
+	})
+
+	memory.Get("/context", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+
+		ctx, err := memoryRepo.GetContextForUser(c.Context(), userID)
+		if err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to get context"})
+		}
+
+		return c.JSON(fiber.Map{"context": ctx})
+	})
+
+	memory.Delete("/:id", func(c *fiber.Ctx) error {
+		memID := c.Params("id")
+
+		if err := memoryRepo.Delete(c.Context(), memID); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to delete memory"})
+		}
+
+		return c.Status(204).Send(nil)
+	})
+
+	pagesAPI := app.Group("/api/v1/pages", middleware.JWT(middleware.JWTConfig{
+		Secret:     cfg.JWTSecret,
+		AuthSvcURL: cfg.AuthSvcURL,
+		AllowGuest: true,
+	}))
+
+	pagesAPI.Get("/", func(c *fiber.Ctx) error {
+		userID := middleware.GetUserID(c)
+		if userID == "" {
+			return c.JSON(fiber.Map{"pages": []interface{}{}})
+		}
+
+		limit := c.QueryInt("limit", 50)
+		offset := c.QueryInt("offset", 0)
+
+		pageList, err := pageRepo.GetByUserID(c.Context(), userID, limit, offset)
+		if err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to get pages"})
+		}
+
+		return c.JSON(fiber.Map{"pages": pageList})
+	})
+
+	pagesAPI.Post("/from-thread/:threadId", func(c *fiber.Ctx) error {
+		threadID := c.Params("threadId")
+		userID := middleware.GetUserID(c)
+
+		if userID == "" {
+			return c.Status(401).JSON(fiber.Map{"error": "Authentication required"})
+		}
+
+		thread, err := threadRepo.GetByID(c.Context(), threadID)
+		if err != nil || thread == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
+		}
+
+		if thread.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		messages, _ := threadRepo.GetMessages(c.Context(), threadID, 100, 0)
+
+		var query, answer string
+		for _, msg := range messages {
+			if msg.Role == "user" && query == "" {
+				query = msg.Content
+			}
+			if msg.Role == "assistant" {
+				answer += msg.Content + "\n\n"
+			}
+		}
+
+		if llmClient == nil {
+			return c.Status(503).JSON(fiber.Map{"error": "LLM not configured"})
+		}
+
+		generator := pages.NewPageGenerator(pages.PageGeneratorConfig{
+			LLMClient: llmClient,
+			Locale:    c.Query("locale", "en"),
+		})
+
+		page, err := generator.GenerateFromThread(c.Context(), query, answer, nil)
+		if err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to generate page"})
+		}
+
+		page.UserID = userID
+		page.ThreadID = threadID
+
+		if err := pageRepo.Create(c.Context(), page); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to save page"})
+		}
+
+		return c.Status(201).JSON(page)
+	})
+
+	pagesAPI.Get("/:id", func(c *fiber.Ctx) error {
+		pageID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		page, err := pageRepo.GetByID(c.Context(), pageID)
+		if err != nil || page == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
+		}
+
+		if page.UserID != userID && !page.IsPublic {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		return c.JSON(page)
+	})
+
+	pagesAPI.Post("/:id/share", func(c *fiber.Ctx) error {
+		pageID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		page, err := pageRepo.GetByID(c.Context(), pageID)
+		if err != nil || page == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
+		}
+
+		if page.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		shareID := generateShareID()
+		if err := pageRepo.SetShareID(c.Context(), pageID, shareID); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to share page"})
+		}
+
+		return c.JSON(fiber.Map{
+			"shareId":  shareID,
+			"shareUrl": fmt.Sprintf("/page/%s", shareID),
+		})
+	})
+
+	pagesAPI.Get("/share/:shareId", func(c *fiber.Ctx) error {
+		shareID := c.Params("shareId")
+
+		page, err := pageRepo.GetByShareID(c.Context(), shareID)
+		if err != nil || page == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
+		}
+
+		pageRepo.IncrementViewCount(c.Context(), page.ID)
+
+		return c.JSON(page)
+	})
+
+	pagesAPI.Delete("/:id", func(c *fiber.Ctx) error {
+		pageID := c.Params("id")
+		userID := middleware.GetUserID(c)
+
+		page, err := pageRepo.GetByID(c.Context(), pageID)
+		if err != nil || page == nil {
+			return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
+		}
+
+		if page.UserID != userID {
+			return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
+		}
+
+		if err := pageRepo.Delete(c.Context(), pageID); err != nil {
+			return c.Status(500).JSON(fiber.Map{"error": "Failed to delete page"})
+		}
+
+		return c.Status(204).Send(nil)
+	})
+
+	port := getEnvInt("THREAD_SVC_PORT", 3027)
+	log.Printf("thread-svc listening on :%d", port)
+	log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
+}
+
+func generateShareID() string {
+	b := make([]byte, 8)
+	rand.Read(b)
+	return hex.EncodeToString(b)
+}
+
+func getEnvInt(key string, defaultValue int) int {
+	if val := os.Getenv(key); val != "" {
+		var result int
+		if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
+			return result
+		}
+	}
+	return defaultValue
+}