gooseek/backend/cmd/discover-svc/main.go

package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"log"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/gofiber/fiber/v2"
	"github.com/gofiber/fiber/v2/middleware/cors"
	"github.com/gofiber/fiber/v2/middleware/logger"
	"github.com/gooseek/backend/internal/db"
	"github.com/gooseek/backend/internal/search"
	"github.com/gooseek/backend/pkg/cache"
	"github.com/gooseek/backend/pkg/config"
	"github.com/gooseek/backend/pkg/middleware"
)

type DigestCitation struct {
	Index  int    `json:"index"`
	URL    string `json:"url"`
	Title  string `json:"title"`
	Domain string `json:"domain"`
}

type Digest struct {
	Topic            string           `json:"topic"`
	Region           string           `json:"region"`
	ClusterTitle     string           `json:"clusterTitle"`
	SummaryRu        string           `json:"summaryRu"`
	Citations        []DigestCitation `json:"citations"`
	SourcesCount     int              `json:"sourcesCount"`
	FollowUp         []string         `json:"followUp"`
	Thumbnail        string           `json:"thumbnail"`
	ShortDescription string           `json:"shortDescription"`
	MainURL          string           `json:"mainUrl"`
	CreatedAt        time.Time        `json:"createdAt"`
}

type ArticleSummary struct {
	URL       string    `json:"url"`
	Events    []string  `json:"events"`
	CreatedAt time.Time `json:"createdAt"`
}

type DiscoverStore struct {
	digests         map[string]*Digest
	articleSummaries map[string]*ArticleSummary
	mu              sync.RWMutex
}

func NewDiscoverStore() *DiscoverStore {
	return &DiscoverStore{
		digests:          make(map[string]*Digest),
		articleSummaries: make(map[string]*ArticleSummary),
	}
}

func (s *DiscoverStore) GetDigest(topic, region, title string) *Digest {
	s.mu.RLock()
	defer s.mu.RUnlock()
	key := fmt.Sprintf("%s:%s:%s", topic, region, title)
	return s.digests[key]
}

func (s *DiscoverStore) GetDigestByURL(url string) *Digest {
	s.mu.RLock()
	defer s.mu.RUnlock()
	normalizedURL := normalizeURL(url)
	for _, d := range s.digests {
		if normalizeURL(d.MainURL) == normalizedURL {
			return d
		}
	}
	return nil
}

func (s *DiscoverStore) GetDigests(topic, region string) []*Digest {
	s.mu.RLock()
	defer s.mu.RUnlock()
	result := make([]*Digest, 0)
	prefix := fmt.Sprintf("%s:%s:", topic, region)
	for k, d := range s.digests {
		if strings.HasPrefix(k, prefix) {
			result = append(result, d)
		}
	}
	return result
}

func (s *DiscoverStore) UpsertDigest(d *Digest) {
	s.mu.Lock()
	defer s.mu.Unlock()
	key := fmt.Sprintf("%s:%s:%s", d.Topic, d.Region, d.ClusterTitle)
	d.CreatedAt = time.Now()
	s.digests[key] = d
}

func (s *DiscoverStore) DeleteDigests(topic, region string) int {
	s.mu.Lock()
	defer s.mu.Unlock()
	count := 0
	prefix := fmt.Sprintf("%s:%s:", topic, region)
	for k := range s.digests {
		if strings.HasPrefix(k, prefix) {
			delete(s.digests, k)
			count++
		}
	}
	return count
}

func (s *DiscoverStore) GetArticleSummary(url string) *ArticleSummary {
	s.mu.RLock()
	defer s.mu.RUnlock()
	key := articleSummaryKey(url)
	return s.articleSummaries[key]
}

func (s *DiscoverStore) SaveArticleSummary(url string, events []string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	key := articleSummaryKey(url)
	s.articleSummaries[key] = &ArticleSummary{
		URL:       url,
		Events:    events,
		CreatedAt: time.Now(),
	}
}

func (s *DiscoverStore) DeleteArticleSummary(url string) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	key := articleSummaryKey(url)
	if _, ok := s.articleSummaries[key]; ok {
		delete(s.articleSummaries, key)
		return true
	}
	return false
}

func articleSummaryKey(url string) string {
	hash := sha256.Sum256([]byte(normalizeURL(url)))
	return hex.EncodeToString(hash[:16])
}

func normalizeURL(url string) string {
	url = strings.TrimSpace(url)
	url = strings.TrimSuffix(url, "/")
	url = strings.TrimPrefix(url, "https://")
	url = strings.TrimPrefix(url, "http://")
	url = strings.TrimPrefix(url, "www.")
	return url
}

func extractDomain(url string) string {
	normalized := normalizeURL(url)
	if idx := strings.Index(normalized, "/"); idx > 0 {
		return normalized[:idx]
	}
	return normalized
}

func main() {
	cfg, err := config.Load()
	if err != nil {
		log.Fatal("Failed to load config:", err)
	}

	store := NewDiscoverStore()
	searchClient := search.NewSearXNGClient(cfg)

	var database *db.PostgresDB
	var digestRepo *db.DigestRepository
	var summaryRepo *db.ArticleSummaryRepository

	if cfg.DatabaseURL != "" {
		database, err = db.NewPostgresDB(cfg.DatabaseURL)
		if err != nil {
			log.Printf("PostgreSQL unavailable: %v (falling back to in-memory)", err)
		} else {
			ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
			if err := database.RunMigrations(ctx); err != nil {
				log.Printf("Migration warning: %v", err)
			}
			cancel()
			defer database.Close()

			digestRepo = db.NewDigestRepository(database)
			summaryRepo = db.NewArticleSummaryRepository(database)
			log.Println("PostgreSQL connected")
		}
	}

	var redisCache *cache.RedisCache
	if cfg.RedisURL != "" {
		redisCache, err = cache.NewRedisCache(cfg.RedisURL, "gooseek:discover")
		if err != nil {
			log.Printf("Redis cache unavailable: %v (falling back to in-memory)", err)
		} else {
			log.Printf("Redis cache connected")
			defer redisCache.Close()
		}
	}

	_ = digestRepo
	_ = summaryRepo

	app := fiber.New(fiber.Config{
		BodyLimit:    100 * 1024 * 1024,
		ReadTimeout:  30 * time.Second,
		WriteTimeout: 30 * time.Second,
		IdleTimeout:  60 * time.Second,
	})

	app.Use(logger.New())
	app.Use(cors.New())

	app.Get("/health", func(c *fiber.Ctx) error {
		return c.JSON(fiber.Map{"status": "ok"})
	})

	app.Get("/ready", func(c *fiber.Ctx) error {
		return c.JSON(fiber.Map{"status": "ready"})
	})

	app.Get("/metrics", func(c *fiber.Ctx) error {
		c.Set("Content-Type", "text/plain; charset=utf-8")
		return c.SendString(
			"# HELP gooseek_up Service is up (1) or down (0)\n" +
				"# TYPE gooseek_up gauge\n" +
				"gooseek_up 1\n",
		)
	})

	discover := app.Group("/api/v1/discover")

	discover.Get("/digest", func(c *fiber.Ctx) error {
		url := c.Query("url")
		if url != "" {
			digest := store.GetDigestByURL(url)
			if digest == nil {
				return c.Status(404).JSON(fiber.Map{"message": "digest not found"})
			}
			return c.JSON(digest)
		}

		topic := c.Query("topic")
		region := c.Query("region")
		title := c.Query("title")

		if topic == "" || region == "" || title == "" {
			return c.Status(400).JSON(fiber.Map{"message": "topic, region, title (or url) required"})
		}

		digest := store.GetDigest(topic, region, title)
		if digest == nil {
			return c.Status(404).JSON(fiber.Map{"message": "digest not found"})
		}

		return c.JSON(digest)
	})

	discoverAuth := app.Group("/api/v1/discover", middleware.JWT(middleware.JWTConfig{
		Secret:     cfg.JWTSecret,
		AuthSvcURL: cfg.AuthSvcURL,
		AllowGuest: false,
	}))

	discoverAuth.Post("/digest", func(c *fiber.Ctx) error {
		var d Digest
		if err := c.BodyParser(&d); err != nil {
			return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
		}

		if d.Topic == "" || d.Region == "" || d.ClusterTitle == "" || d.SummaryRu == "" {
			return c.Status(400).JSON(fiber.Map{"message": "topic, region, clusterTitle, summaryRu required"})
		}

		store.UpsertDigest(&d)
		return c.Status(204).Send(nil)
	})

	discoverAuth.Delete("/digest", func(c *fiber.Ctx) error {
		topic := c.Query("topic")
		region := c.Query("region")

		if topic == "" || region == "" {
			return c.Status(400).JSON(fiber.Map{"message": "topic, region required"})
		}

		deleted := store.DeleteDigests(topic, region)
		return c.JSON(fiber.Map{"deleted": deleted})
	})

	discover.Get("/article-summary", func(c *fiber.Ctx) error {
		url := c.Query("url")
		if url == "" {
			return c.Status(400).JSON(fiber.Map{"message": "url required"})
		}

		if redisCache != nil {
			events, err := redisCache.GetCachedArticleSummary(c.Context(), url)
			if err == nil && len(events) > 0 {
				return c.JSON(fiber.Map{"events": events})
			}
		}

		if summaryRepo != nil {
			summary, err := summaryRepo.GetByURL(c.Context(), url)
			if err == nil && summary != nil {
				if redisCache != nil {
					redisCache.CacheArticleSummary(c.Context(), url, summary.Events, 24*time.Hour)
				}
				return c.JSON(fiber.Map{"events": summary.Events})
			}
		}

		summary := store.GetArticleSummary(url)
		if summary == nil {
			return c.Status(404).JSON(fiber.Map{"message": "not found"})
		}

		return c.JSON(fiber.Map{"events": summary.Events})
	})

	discoverAuth.Post("/article-summary", func(c *fiber.Ctx) error {
		var body struct {
			URL    string   `json:"url"`
			Events []string `json:"events"`
		}

		if err := c.BodyParser(&body); err != nil {
			return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
		}

		if body.URL == "" || len(body.Events) == 0 {
			return c.Status(400).JSON(fiber.Map{"message": "url and events[] required"})
		}

		store.SaveArticleSummary(body.URL, body.Events)

		if summaryRepo != nil {
			ttl := 7 * 24 * time.Hour
			if err := summaryRepo.Save(c.Context(), body.URL, body.Events, ttl); err != nil {
				log.Printf("postgres save article-summary error: %v", err)
			}
		}

		if redisCache != nil {
			ttl := 24 * time.Hour
			if err := redisCache.CacheArticleSummary(c.Context(), body.URL, body.Events, ttl); err != nil {
				log.Printf("redis cache article-summary error: %v", err)
			}
		}

		log.Printf("article-summary saved: %s (%d events)", body.URL[:min(60, len(body.URL))], len(body.Events))
		return c.Status(204).Send(nil)
	})

	discoverAuth.Delete("/article-summary", func(c *fiber.Ctx) error {
		url := c.Query("url")
		if url == "" {
			return c.Status(400).JSON(fiber.Map{"message": "url required"})
		}

		deleted := store.DeleteArticleSummary(url)
		log.Printf("article-summary deleted: %s (deleted=%v)", url[:min(60, len(url))], deleted)
		return c.Status(204).Send(nil)
	})

	discover.Get("/search", func(c *fiber.Ctx) error {
		q := c.Query("q")
		if q == "" {
			return c.Status(400).JSON(fiber.Map{"message": "Query q is required"})
		}

		ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout)
		defer cancel()

		result, err := searchClient.Search(ctx, q, &search.SearchOptions{PageNo: 1})
		if err != nil {
			return c.Status(503).JSON(fiber.Map{"message": "Search failed"})
		}

		if len(result.Results) > 10 {
			result.Results = result.Results[:10]
		}

		return c.JSON(fiber.Map{"results": result.Results})
	})

	discover.Get("/", func(c *fiber.Ctx) error {
		topic := c.Query("topic", "tech")
		region := c.Query("region", "world")
		page := c.QueryInt("page", 1)
		limit := c.QueryInt("limit", 10)

		if page < 1 {
			page = 1
		}
		if limit < 1 || limit > 30 {
			limit = 10
		}

		digests := store.GetDigests(topic, region)
		if len(digests) > 0 {
			start := (page - 1) * limit
			end := start + limit
			if start >= len(digests) {
				return c.JSON(fiber.Map{
					"blogs":   []fiber.Map{},
					"hasMore": false,
					"page":    page,
					"total":   len(digests),
				})
			}
			if end > len(digests) {
				end = len(digests)
			}

			pagedDigests := digests[start:end]
			blogs := make([]fiber.Map, len(pagedDigests))
			for i, d := range pagedDigests {
				content := d.ShortDescription
				if content == "" && len(d.SummaryRu) > 200 {
					content = d.SummaryRu[:200] + "…"
				} else if content == "" {
					content = d.SummaryRu
				}

				blogs[i] = fiber.Map{
					"title":        d.ClusterTitle,
					"content":      content,
					"url":          d.MainURL,
					"thumbnail":    d.Thumbnail,
					"sourcesCount": d.SourcesCount,
					"digestId":     fmt.Sprintf("%s:%s:%s", d.Topic, d.Region, d.ClusterTitle),
				}
			}
			return c.JSON(fiber.Map{
				"blogs":   blogs,
				"hasMore": end < len(digests),
				"page":    page,
				"total":   len(digests),
			})
		}

		ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout*2)
		defer cancel()

		queries := getQueriesForTopic(topic, region)
		results, err := searchClient.Search(ctx, queries[0], &search.SearchOptions{
			Categories: []string{"news"},
			PageNo:     page,
		})
		if err != nil {
			return c.Status(503).JSON(fiber.Map{"message": "Search failed"})
		}

		blogs := make([]fiber.Map, 0, limit)
		for i, r := range results.Results {
			if i >= limit {
				break
			}
			thumbnail := r.Thumbnail
			if thumbnail == "" {
				thumbnail = r.ThumbnailSrc
			}
			if thumbnail == "" {
				thumbnail = r.ImgSrc
			}

			content := r.Content
			if content == "" {
				content = r.Title
			}
			if len(content) > 300 {
				content = content[:300] + "…"
			}

			blogs = append(blogs, fiber.Map{
				"title":     r.Title,
				"content":   content,
				"url":       r.URL,
				"thumbnail": thumbnail,
			})
		}

		hasMore := len(results.Results) > limit
		return c.JSON(fiber.Map{
			"blogs":   blogs,
			"hasMore": hasMore,
			"page":    page,
		})
	})

	port := getEnvInt("DISCOVER_SVC_PORT", 3002)
	log.Printf("discover-svc listening on :%d", port)
	log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
}

func getQueriesForTopic(topic, region string) []string {
	queries := map[string]map[string][]string{
		"tech": {
			"world":  {"technology news AI innovation"},
			"russia": {"технологии новости IT инновации искусственный интеллект"},
			"eu":     {"technology news Europe AI"},
		},
		"finance": {
			"world":  {"finance news economy markets stocks"},
			"russia": {"финансы новости экономика рынки акции"},
			"eu":     {"finance news Europe economy markets"},
		},
		"sports": {
			"world":  {"sports news football basketball Olympics"},
			"russia": {"спорт новости футбол хоккей КХЛ РПЛ"},
			"eu":     {"sports news football Champions League"},
		},
		"politics": {
			"world":  {"politics news government elections policy"},
			"russia": {"политика новости Россия правительство законы"},
			"eu":     {"politics news Europe EU parliament"},
		},
		"science": {
			"world":  {"science news research discovery space"},
			"russia": {"наука новости исследования открытия космос"},
			"eu":     {"science news Europe research discovery"},
		},
		"health": {
			"world":  {"health news medicine medical research"},
			"russia": {"здоровье новости медицина лечение"},
			"eu":     {"health news Europe medicine healthcare"},
		},
		"entertainment": {
			"world":  {"entertainment news movies music celebrities"},
			"russia": {"развлечения новости кино музыка шоу-бизнес"},
			"eu":     {"entertainment news Europe movies music"},
		},
		"world": {
			"world":  {"world news international global events"},
			"russia": {"мировые новости международные события"},
			"eu":     {"world news Europe international"},
		},
		"business": {
			"world":  {"business news companies startups industry"},
			"russia": {"бизнес новости компании стартапы предпринимательство"},
			"eu":     {"business news Europe companies industry"},
		},
		"culture": {
			"world":  {"culture news art exhibitions theatre"},
			"russia": {"культура новости искусство выставки театр"},
			"eu":     {"culture news Europe art exhibitions"},
		},
	}

	if topicQueries, ok := queries[topic]; ok {
		if regionQueries, ok := topicQueries[region]; ok {
			return regionQueries
		}
		if defaultQueries, ok := topicQueries["world"]; ok {
			return defaultQueries
		}
	}

	return []string{"news today"}
}

func getEnvInt(key string, defaultValue int) int {
	if val := os.Getenv(key); val != "" {
		var result int
		if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
			return result
		}
	}
	return defaultValue
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}