feat: Go backend, enhanced search, new widgets, Docker deploy
Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
This commit is contained in:
home
209
backend/cmd/agent-svc/main.go
Normal file
209
backend/cmd/agent-svc/main.go
Normal file
@@ -0,0 +1,209 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/agent"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/internal/search"
|
||||
"github.com/gooseek/backend/internal/session"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/gooseek/backend/pkg/ndjson"
|
||||
)
|
||||
|
||||
type SearchRequest struct {
|
||||
Message struct {
|
||||
MessageID string `json:"messageId"`
|
||||
ChatID string `json:"chatId"`
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
OptimizationMode string `json:"optimizationMode"`
|
||||
Sources []string `json:"sources"`
|
||||
History [][]string `json:"history"`
|
||||
Files []string `json:"files"`
|
||||
ChatModel ChatModel `json:"chatModel"`
|
||||
SystemInstructions string `json:"systemInstructions"`
|
||||
Locale string `json:"locale"`
|
||||
AnswerMode string `json:"answerMode"`
|
||||
ResponsePrefs *struct {
|
||||
Format string `json:"format"`
|
||||
Length string `json:"length"`
|
||||
Tone string `json:"tone"`
|
||||
} `json:"responsePrefs"`
|
||||
LearningMode bool `json:"learningMode"`
|
||||
}
|
||||
|
||||
type ChatModel struct {
|
||||
ProviderID string `json:"providerId"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
searchClient := search.NewSearXNGClient(cfg)
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
StreamRequestBody: true,
|
||||
BodyLimit: 10 * 1024 * 1024,
|
||||
ReadTimeout: time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
IdleTimeout: 2 * time.Minute,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/agents/search", func(c *fiber.Ctx) error {
|
||||
var req SearchRequest
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if req.Message.Content == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Message content required"})
|
||||
}
|
||||
|
||||
providerID := req.ChatModel.ProviderID
|
||||
modelKey := req.ChatModel.Key
|
||||
|
||||
if providerID == "" && cfg.TimewebAPIKey != "" {
|
||||
providerID = "timeweb"
|
||||
modelKey = "gpt-4o"
|
||||
} else if providerID == "" {
|
||||
providerID = "openai"
|
||||
modelKey = "gpt-4o-mini"
|
||||
}
|
||||
|
||||
llmClient, err := llm.NewClient(llm.ProviderConfig{
|
||||
ProviderID: providerID,
|
||||
ModelKey: modelKey,
|
||||
APIKey: getAPIKey(cfg, providerID),
|
||||
BaseURL: cfg.TimewebAPIBaseURL,
|
||||
AgentAccessID: cfg.TimewebAgentAccessID,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to create LLM client: " + err.Error()})
|
||||
}
|
||||
|
||||
chatHistory := make([]llm.Message, 0, len(req.History))
|
||||
for _, h := range req.History {
|
||||
if len(h) >= 2 {
|
||||
role := llm.RoleUser
|
||||
if h[0] == "ai" || h[0] == "assistant" {
|
||||
role = llm.RoleAssistant
|
||||
}
|
||||
chatHistory = append(chatHistory, llm.Message{
|
||||
Role: role,
|
||||
Content: h[1],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
mode := agent.ModeBalanced
|
||||
switch req.OptimizationMode {
|
||||
case "speed":
|
||||
mode = agent.ModeSpeed
|
||||
case "quality":
|
||||
mode = agent.ModeQuality
|
||||
}
|
||||
|
||||
var responsePrefs *agent.ResponsePrefs
|
||||
if req.ResponsePrefs != nil {
|
||||
responsePrefs = &agent.ResponsePrefs{
|
||||
Format: req.ResponsePrefs.Format,
|
||||
Length: req.ResponsePrefs.Length,
|
||||
Tone: req.ResponsePrefs.Tone,
|
||||
}
|
||||
}
|
||||
|
||||
input := agent.OrchestratorInput{
|
||||
ChatHistory: chatHistory,
|
||||
FollowUp: req.Message.Content,
|
||||
Config: agent.OrchestratorConfig{
|
||||
LLM: llmClient,
|
||||
SearchClient: searchClient,
|
||||
Mode: mode,
|
||||
Sources: req.Sources,
|
||||
FileIDs: req.Files,
|
||||
SystemInstructions: req.SystemInstructions,
|
||||
Locale: req.Locale,
|
||||
AnswerMode: req.AnswerMode,
|
||||
ResponsePrefs: responsePrefs,
|
||||
LearningMode: req.LearningMode,
|
||||
DiscoverSvcURL: cfg.DiscoverSvcURL,
|
||||
Crawl4AIURL: cfg.Crawl4AIURL,
|
||||
},
|
||||
}
|
||||
|
||||
sess := session.NewSession()
|
||||
|
||||
c.Set("Content-Type", "application/x-ndjson")
|
||||
c.Set("Cache-Control", "no-cache")
|
||||
c.Set("Transfer-Encoding", "chunked")
|
||||
|
||||
c.Context().SetBodyStreamWriter(func(w *bufio.Writer) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
writer := ndjson.NewWriter(w)
|
||||
|
||||
unsubscribe := sess.Subscribe(func(eventType session.EventType, data interface{}) {
|
||||
if eventType == session.EventData {
|
||||
if dataMap, ok := data.(map[string]interface{}); ok {
|
||||
writer.Write(dataMap)
|
||||
w.Flush()
|
||||
}
|
||||
}
|
||||
})
|
||||
defer unsubscribe()
|
||||
|
||||
err := agent.RunOrchestrator(ctx, sess, input)
|
||||
if err != nil {
|
||||
ndjson.WriteError(writer, err)
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
port := cfg.AgentSvcPort
|
||||
log.Printf("agent-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getAPIKey(cfg *config.Config, providerID string) string {
|
||||
switch providerID {
|
||||
case "timeweb":
|
||||
return cfg.TimewebAPIKey
|
||||
case "openai":
|
||||
return cfg.OpenAIAPIKey
|
||||
case "anthropic":
|
||||
return cfg.AnthropicAPIKey
|
||||
case "gemini", "google":
|
||||
return cfg.GeminiAPIKey
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
if os.Getenv("PORT") == "" {
|
||||
os.Setenv("PORT", "3018")
|
||||
}
|
||||
}
|
||||
232
backend/cmd/api-gateway/main.go
Normal file
232
backend/cmd/api-gateway/main.go
Normal file
@@ -0,0 +1,232 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
)
|
||||
|
||||
var svcURLs map[string]string
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
svcURLs = map[string]string{
|
||||
"chat": cfg.ChatSvcURL,
|
||||
"agents": cfg.AgentSvcURL,
|
||||
"search": cfg.SearchSvcURL,
|
||||
"llm": cfg.LLMSvcURL,
|
||||
"scraper": cfg.ScraperSvcURL,
|
||||
"memory": cfg.MemorySvcURL,
|
||||
"library": cfg.LibrarySvcURL,
|
||||
"thread": cfg.ThreadSvcURL,
|
||||
"discover": cfg.DiscoverSvcURL,
|
||||
"finance": cfg.FinanceHeatmapURL,
|
||||
"learning": cfg.LearningSvcURL,
|
||||
"computer": cfg.ComputerSvcURL,
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
StreamRequestBody: true,
|
||||
BodyLimit: 50 * 1024 * 1024,
|
||||
ReadTimeout: time.Duration(cfg.HTTPTimeout),
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
IdleTimeout: 2 * time.Minute,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New(cors.Config{
|
||||
AllowOrigins: strings.Join(cfg.AllowedOrigins, ","),
|
||||
AllowHeaders: "Origin, Content-Type, Accept, Authorization",
|
||||
AllowMethods: "GET, POST, PUT, PATCH, DELETE, OPTIONS",
|
||||
}))
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/ready", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ready"})
|
||||
})
|
||||
|
||||
app.Post("/api/chat", handleChat)
|
||||
app.All("/api/*", handleProxy)
|
||||
|
||||
port := cfg.APIGatewayPort
|
||||
log.Printf("api-gateway listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getTarget(path string) (base, rewrite string) {
|
||||
switch {
|
||||
case path == "/api/chat" || strings.HasPrefix(path, "/api/chat?"):
|
||||
return svcURLs["chat"], "/api/v1/chat"
|
||||
case strings.HasPrefix(path, "/api/v1/agents"):
|
||||
return svcURLs["agents"], path
|
||||
case strings.HasPrefix(path, "/api/v1/search"):
|
||||
return svcURLs["search"], path
|
||||
case strings.HasPrefix(path, "/api/v1/llm"), strings.HasPrefix(path, "/api/v1/providers"):
|
||||
return svcURLs["llm"], path
|
||||
case strings.HasPrefix(path, "/api/v1/memory"):
|
||||
return svcURLs["memory"], path
|
||||
case strings.HasPrefix(path, "/api/v1/library"):
|
||||
return svcURLs["library"], path
|
||||
case strings.HasPrefix(path, "/api/v1/threads"):
|
||||
return svcURLs["thread"], path
|
||||
case strings.HasPrefix(path, "/api/v1/spaces"):
|
||||
return svcURLs["thread"], path
|
||||
case strings.HasPrefix(path, "/api/v1/pages"):
|
||||
return svcURLs["thread"], path
|
||||
case strings.HasPrefix(path, "/api/v1/share"):
|
||||
return svcURLs["thread"], path
|
||||
case strings.HasPrefix(path, "/api/v1/discover"):
|
||||
return svcURLs["discover"], path
|
||||
case strings.HasPrefix(path, "/api/v1/heatmap"):
|
||||
return svcURLs["finance"], path
|
||||
case strings.HasPrefix(path, "/api/v1/movers"):
|
||||
return svcURLs["finance"], path
|
||||
case strings.HasPrefix(path, "/api/v1/markets"):
|
||||
return svcURLs["finance"], path
|
||||
case strings.HasPrefix(path, "/api/v1/learning"):
|
||||
return svcURLs["learning"], path
|
||||
case strings.HasPrefix(path, "/api/v1/computer"):
|
||||
return svcURLs["computer"], path
|
||||
default:
|
||||
return "", ""
|
||||
}
|
||||
}
|
||||
|
||||
func handleChat(c *fiber.Ctx) error {
|
||||
base := svcURLs["chat"]
|
||||
if base == "" {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Chat service not configured"})
|
||||
}
|
||||
|
||||
targetURL := strings.TrimSuffix(base, "/") + "/api/v1/chat"
|
||||
|
||||
req, err := http.NewRequest("POST", targetURL, strings.NewReader(string(c.Body())))
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if auth := c.Get("Authorization"); auth != "" {
|
||||
req.Header.Set("Authorization", auth)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 5 * time.Minute}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return c.Status(resp.StatusCode).Send(body)
|
||||
}
|
||||
|
||||
c.Set("Content-Type", "application/x-ndjson")
|
||||
c.Set("Cache-Control", "no-cache")
|
||||
c.Set("Transfer-Encoding", "chunked")
|
||||
|
||||
c.Context().SetBodyStreamWriter(func(w *bufio.Writer) {
|
||||
defer resp.Body.Close()
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := resp.Body.Read(buf)
|
||||
if n > 0 {
|
||||
w.Write(buf[:n])
|
||||
w.Flush()
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleProxy(c *fiber.Ctx) error {
|
||||
path := c.Path()
|
||||
base, rewrite := getTarget(path)
|
||||
|
||||
if base == "" {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Not found"})
|
||||
}
|
||||
|
||||
targetURL := strings.TrimSuffix(base, "/") + rewrite
|
||||
if c.Context().QueryArgs().Len() > 0 {
|
||||
targetURL += "?" + string(c.Context().QueryArgs().QueryString())
|
||||
}
|
||||
|
||||
method := c.Method()
|
||||
var body io.Reader
|
||||
if method != "GET" && method != "HEAD" {
|
||||
body = strings.NewReader(string(c.Body()))
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(method, targetURL, body)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
passHeaders := []string{"Authorization", "Content-Type", "Accept", "User-Agent", "Accept-Language"}
|
||||
for _, h := range passHeaders {
|
||||
if v := c.Get(h); v != "" {
|
||||
req.Header.Set(h, v)
|
||||
}
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: time.Minute}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return handleFallback(c, path)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
for _, h := range []string{"Content-Type", "Cache-Control", "Set-Cookie"} {
|
||||
if v := resp.Header.Get(h); v != "" {
|
||||
c.Set(h, v)
|
||||
}
|
||||
}
|
||||
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
return c.Status(resp.StatusCode).Send(data)
|
||||
}
|
||||
|
||||
func handleFallback(c *fiber.Ctx, path string) error {
|
||||
switch {
|
||||
case strings.HasPrefix(path, "/api/v1/discover"):
|
||||
return c.JSON(fiber.Map{"items": []interface{}{}})
|
||||
case strings.HasPrefix(path, "/api/geo-context"):
|
||||
return c.JSON(fiber.Map{"country": nil, "city": nil})
|
||||
case strings.HasPrefix(path, "/api/translations"):
|
||||
return c.JSON(fiber.Map{})
|
||||
default:
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
if os.Getenv("PORT") == "" {
|
||||
os.Setenv("PORT", "3015")
|
||||
}
|
||||
}
|
||||
53
backend/cmd/browser-svc/main.go
Normal file
53
backend/cmd/browser-svc/main.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gooseek/backend/internal/computer/browser"
|
||||
)
|
||||
|
||||
func main() {
|
||||
port := 3050
|
||||
if p := os.Getenv("PORT"); p != "" {
|
||||
if parsed, err := strconv.Atoi(p); err == nil {
|
||||
port = parsed
|
||||
}
|
||||
}
|
||||
if p := os.Getenv("BROWSER_SVC_PORT"); p != "" {
|
||||
if parsed, err := strconv.Atoi(p); err == nil {
|
||||
port = parsed
|
||||
}
|
||||
}
|
||||
|
||||
cfg := browser.ServerConfig{
|
||||
Port: port,
|
||||
MaxSessions: 20,
|
||||
SessionTimeout: 30 * time.Minute,
|
||||
CleanupInterval: 5 * time.Minute,
|
||||
}
|
||||
|
||||
server := browser.NewBrowserServer(cfg)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
go func() {
|
||||
<-sigCh
|
||||
log.Println("[browser-svc] Shutting down...")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
log.Printf("[browser-svc] Starting browser service on port %d", port)
|
||||
if err := server.Start(ctx); err != nil {
|
||||
log.Fatalf("[browser-svc] Server error: %v", err)
|
||||
}
|
||||
}
|
||||
235
backend/cmd/chat-svc/main.go
Normal file
235
backend/cmd/chat-svc/main.go
Normal file
@@ -0,0 +1,235 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
)
|
||||
|
||||
type ChatRequest struct {
|
||||
Message struct {
|
||||
MessageID string `json:"messageId"`
|
||||
ChatID string `json:"chatId"`
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
OptimizationMode string `json:"optimizationMode"`
|
||||
Sources []string `json:"sources"`
|
||||
History [][]string `json:"history"`
|
||||
Files []string `json:"files"`
|
||||
ChatModel ChatModel `json:"chatModel"`
|
||||
EmbeddingModel ChatModel `json:"embeddingModel"`
|
||||
SystemInstructions string `json:"systemInstructions"`
|
||||
Locale string `json:"locale"`
|
||||
AnswerMode string `json:"answerMode"`
|
||||
ResponsePrefs *struct {
|
||||
Format string `json:"format"`
|
||||
Length string `json:"length"`
|
||||
Tone string `json:"tone"`
|
||||
} `json:"responsePrefs"`
|
||||
LearningMode bool `json:"learningMode"`
|
||||
}
|
||||
|
||||
type ChatModel struct {
|
||||
ProviderID string `json:"providerId"`
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
StreamRequestBody: true,
|
||||
BodyLimit: 50 * 1024 * 1024,
|
||||
ReadTimeout: time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
IdleTimeout: 2 * time.Minute,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/ready", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ready"})
|
||||
})
|
||||
|
||||
app.Get("/metrics", func(c *fiber.Ctx) error {
|
||||
c.Set("Content-Type", "text/plain; charset=utf-8")
|
||||
return c.SendString(
|
||||
"# HELP gooseek_up Service is up (1) or down (0)\n" +
|
||||
"# TYPE gooseek_up gauge\n" +
|
||||
"gooseek_up 1\n",
|
||||
)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/config", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{
|
||||
"values": fiber.Map{
|
||||
"version": 1,
|
||||
"setupComplete": true,
|
||||
"preferences": fiber.Map{},
|
||||
},
|
||||
"fields": fiber.Map{},
|
||||
"modelProviders": []interface{}{},
|
||||
"envOnlyMode": true,
|
||||
})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/chat", func(c *fiber.Ctx) error {
|
||||
var req ChatRequest
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if req.Message.Content == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Message content required"})
|
||||
}
|
||||
|
||||
isDiscoverSummary := strings.HasPrefix(req.Message.Content, "Summary: ") && len(req.Message.Content) > 9
|
||||
var summaryURL string
|
||||
if isDiscoverSummary {
|
||||
summaryURL = strings.TrimSpace(strings.TrimPrefix(req.Message.Content, "Summary: "))
|
||||
}
|
||||
|
||||
agentURL := strings.TrimSuffix(cfg.AgentSvcURL, "/") + "/api/v1/agents/search"
|
||||
|
||||
httpReq, err := http.NewRequest("POST", agentURL, bytes.NewReader(c.Body()))
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
httpReq.Header.Set("Content-Type", "application/json")
|
||||
if auth := c.Get("Authorization"); auth != "" {
|
||||
httpReq.Header.Set("Authorization", auth)
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 5 * time.Minute}
|
||||
resp, err := client.Do(httpReq)
|
||||
if err != nil {
|
||||
log.Printf("Agent service error: %v", err)
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Agent service unavailable"})
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return c.Status(resp.StatusCode).Send(body)
|
||||
}
|
||||
|
||||
c.Set("Content-Type", "application/x-ndjson")
|
||||
c.Set("Cache-Control", "no-cache")
|
||||
c.Set("Connection", "keep-alive")
|
||||
|
||||
if isDiscoverSummary && summaryURL != "" && cfg.DiscoverSvcURL != "" {
|
||||
collected := make([]string, 0)
|
||||
scanner := bufio.NewScanner(resp.Body)
|
||||
scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
|
||||
|
||||
var result bytes.Buffer
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.TrimSpace(line) != "" {
|
||||
collected = append(collected, line)
|
||||
result.WriteString(line + "\n")
|
||||
}
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if len(collected) > 0 {
|
||||
go saveArticleSummary(cfg.DiscoverSvcURL, summaryURL, collected)
|
||||
}
|
||||
return c.SendString(result.String())
|
||||
}
|
||||
|
||||
return c.SendStream(resp.Body)
|
||||
})
|
||||
|
||||
port := cfg.ChatSvcPort
|
||||
log.Printf("chat-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func init() {
|
||||
if os.Getenv("PORT") == "" {
|
||||
os.Setenv("PORT", "3005")
|
||||
}
|
||||
}
|
||||
|
||||
func saveArticleSummary(discoverSvcURL, articleURL string, events []string) {
|
||||
if discoverSvcURL == "" || articleURL == "" || len(events) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"url": articleURL,
|
||||
"events": events,
|
||||
}
|
||||
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
log.Printf("article-summary marshal error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
url := strings.TrimSuffix(discoverSvcURL, "/") + "/api/v1/discover/article-summary"
|
||||
|
||||
maxRetries := 5
|
||||
retryDelay := 2 * time.Second
|
||||
|
||||
for attempt := 1; attempt <= maxRetries; attempt++ {
|
||||
req, err := http.NewRequest("POST", url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
log.Printf("article-summary request error (attempt %d): %v", attempt, err)
|
||||
continue
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 2 * time.Minute}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("article-summary save error (attempt %d): %v", attempt, err)
|
||||
if attempt < maxRetries {
|
||||
time.Sleep(retryDelay)
|
||||
}
|
||||
continue
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
log.Printf("article-summary saved: %s", articleURL[:min(60, len(articleURL))])
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("article-summary save failed (attempt %d): status %d", attempt, resp.StatusCode)
|
||||
if attempt < maxRetries {
|
||||
time.Sleep(retryDelay)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("article-summary save failed after %d retries: %s", maxRetries, articleURL[:min(60, len(articleURL))])
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
345
backend/cmd/collection-svc/main.go
Normal file
345
backend/cmd/collection-svc/main.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/db"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/gooseek/backend/pkg/middleware"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
var database *db.PostgresDB
|
||||
var collectionRepo *db.CollectionRepository
|
||||
|
||||
if cfg.DatabaseURL != "" {
|
||||
database, err = db.NewPostgresDB(cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Printf("Database unavailable: %v (some features disabled)", err)
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if err := database.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Migration warning: %v", err)
|
||||
}
|
||||
cancel()
|
||||
defer database.Close()
|
||||
|
||||
collectionRepo = db.NewCollectionRepository(database)
|
||||
log.Println("PostgreSQL connected")
|
||||
}
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 50 * 1024 * 1024,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
if cfg.JWTSecret != "" || cfg.AuthSvcURL != "" {
|
||||
app.Use(middleware.JWT(middleware.JWTConfig{
|
||||
Secret: cfg.JWTSecret,
|
||||
AuthSvcURL: cfg.AuthSvcURL,
|
||||
AllowGuest: false,
|
||||
SkipPaths: []string{"/health", "/ready"},
|
||||
}))
|
||||
}
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/ready", func(c *fiber.Ctx) error {
|
||||
if database == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"status": "database unavailable"})
|
||||
}
|
||||
return c.JSON(fiber.Map{"status": "ready"})
|
||||
})
|
||||
|
||||
api := app.Group("/api/v1/collections")
|
||||
|
||||
api.Get("/", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Unauthorized"})
|
||||
}
|
||||
|
||||
limit := c.QueryInt("limit", 50)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
collections, err := collectionRepo.GetByUserID(c.Context(), userID, limit, offset)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get collections"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"collections": collections})
|
||||
})
|
||||
|
||||
api.Post("/", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Unauthorized"})
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
IsPublic bool `json:"isPublic"`
|
||||
ContextEnabled bool `json:"contextEnabled"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if req.Name == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Name is required"})
|
||||
}
|
||||
|
||||
collection := &db.Collection{
|
||||
UserID: userID,
|
||||
Name: req.Name,
|
||||
Description: req.Description,
|
||||
IsPublic: req.IsPublic,
|
||||
ContextEnabled: req.ContextEnabled,
|
||||
}
|
||||
|
||||
if err := collectionRepo.Create(c.Context(), collection); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to create collection"})
|
||||
}
|
||||
|
||||
return c.Status(201).JSON(collection)
|
||||
})
|
||||
|
||||
api.Get("/:id", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
collectionID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
collection, err := collectionRepo.GetByID(c.Context(), collectionID)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get collection"})
|
||||
}
|
||||
if collection == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Collection not found"})
|
||||
}
|
||||
|
||||
if collection.UserID != userID && !collection.IsPublic {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
items, err := collectionRepo.GetItems(c.Context(), collectionID)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get items"})
|
||||
}
|
||||
collection.Items = items
|
||||
|
||||
return c.JSON(collection)
|
||||
})
|
||||
|
||||
api.Put("/:id", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
collectionID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
collection, err := collectionRepo.GetByID(c.Context(), collectionID)
|
||||
if err != nil || collection == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Collection not found"})
|
||||
}
|
||||
|
||||
if collection.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
IsPublic bool `json:"isPublic"`
|
||||
ContextEnabled bool `json:"contextEnabled"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
collection.Name = req.Name
|
||||
collection.Description = req.Description
|
||||
collection.IsPublic = req.IsPublic
|
||||
collection.ContextEnabled = req.ContextEnabled
|
||||
|
||||
if err := collectionRepo.Update(c.Context(), collection); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to update collection"})
|
||||
}
|
||||
|
||||
return c.JSON(collection)
|
||||
})
|
||||
|
||||
api.Delete("/:id", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
collectionID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
collection, err := collectionRepo.GetByID(c.Context(), collectionID)
|
||||
if err != nil || collection == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Collection not found"})
|
||||
}
|
||||
|
||||
if collection.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
if err := collectionRepo.Delete(c.Context(), collectionID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to delete collection"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
api.Post("/:id/items", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
collectionID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
collection, err := collectionRepo.GetByID(c.Context(), collectionID)
|
||||
if err != nil || collection == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Collection not found"})
|
||||
}
|
||||
|
||||
if collection.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
var req struct {
|
||||
ItemType string `json:"itemType"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
URL string `json:"url"`
|
||||
Metadata map[string]interface{} `json:"metadata"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if req.ItemType == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "itemType is required"})
|
||||
}
|
||||
|
||||
item := &db.CollectionItem{
|
||||
CollectionID: collectionID,
|
||||
ItemType: req.ItemType,
|
||||
Title: req.Title,
|
||||
Content: req.Content,
|
||||
URL: req.URL,
|
||||
Metadata: req.Metadata,
|
||||
}
|
||||
|
||||
if err := collectionRepo.AddItem(c.Context(), item); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to add item"})
|
||||
}
|
||||
|
||||
return c.Status(201).JSON(item)
|
||||
})
|
||||
|
||||
api.Delete("/:id/items/:itemId", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
collectionID := c.Params("id")
|
||||
itemID := c.Params("itemId")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
collection, err := collectionRepo.GetByID(c.Context(), collectionID)
|
||||
if err != nil || collection == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Collection not found"})
|
||||
}
|
||||
|
||||
if collection.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
if err := collectionRepo.RemoveItem(c.Context(), itemID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to remove item"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
api.Get("/:id/context", func(c *fiber.Ctx) error {
|
||||
if collectionRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Database unavailable"})
|
||||
}
|
||||
|
||||
collectionID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
collection, err := collectionRepo.GetByID(c.Context(), collectionID)
|
||||
if err != nil || collection == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Collection not found"})
|
||||
}
|
||||
|
||||
if collection.UserID != userID && !collection.IsPublic {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
if !collection.ContextEnabled {
|
||||
return c.JSON(fiber.Map{"context": "", "enabled": false})
|
||||
}
|
||||
|
||||
context, err := collectionRepo.GetCollectionContext(c.Context(), collectionID)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get context"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"context": context, "enabled": true})
|
||||
})
|
||||
|
||||
port := getEnvInt("COLLECTION_SVC_PORT", 3025)
|
||||
log.Printf("collection-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
552
backend/cmd/computer-svc/main.go
Normal file
552
backend/cmd/computer-svc/main.go
Normal file
@@ -0,0 +1,552 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gooseek/backend/internal/computer"
|
||||
"github.com/gooseek/backend/internal/computer/connectors"
|
||||
"github.com/gooseek/backend/internal/db"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/gooseek/backend/pkg/middleware"
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/recover"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load config: %v", err)
|
||||
}
|
||||
|
||||
var database *db.PostgresDB
|
||||
maxRetries := 30
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
database, err = db.NewPostgresDB(cfg.DatabaseURL)
|
||||
if err == nil {
|
||||
log.Println("PostgreSQL connected successfully")
|
||||
break
|
||||
}
|
||||
log.Printf("Waiting for database (attempt %d/%d): %v", i+1, maxRetries, err)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to database after %d attempts: %v", maxRetries, err)
|
||||
}
|
||||
|
||||
taskRepo := db.NewComputerTaskRepo(database.DB())
|
||||
memoryRepo := db.NewComputerMemoryRepo(database.DB())
|
||||
artifactRepo := db.NewComputerArtifactRepo(database.DB())
|
||||
|
||||
if err := taskRepo.Migrate(); err != nil {
|
||||
log.Printf("Task repo migration warning: %v", err)
|
||||
}
|
||||
if err := memoryRepo.Migrate(); err != nil {
|
||||
log.Printf("Memory repo migration warning: %v", err)
|
||||
}
|
||||
if err := artifactRepo.Migrate(); err != nil {
|
||||
log.Printf("Artifact repo migration warning: %v", err)
|
||||
}
|
||||
|
||||
registry := llm.NewModelRegistry()
|
||||
setupModels(registry, cfg)
|
||||
|
||||
connectorHub := connectors.NewConnectorHub()
|
||||
setupConnectors(connectorHub, cfg)
|
||||
|
||||
comp := computer.NewComputer(computer.ComputerConfig{
|
||||
MaxParallelTasks: 10,
|
||||
MaxSubTasks: 20,
|
||||
TaskTimeout: 30 * time.Minute,
|
||||
SubTaskTimeout: 5 * time.Minute,
|
||||
TotalBudget: 1.0,
|
||||
EnableSandbox: true,
|
||||
EnableScheduling: true,
|
||||
SandboxImage: getEnv("SANDBOX_IMAGE", "gooseek/sandbox:latest"),
|
||||
}, computer.Dependencies{
|
||||
Registry: registry,
|
||||
TaskRepo: taskRepo,
|
||||
MemoryRepo: memoryRepo,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
comp.StartScheduler(ctx)
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
ErrorHandler: func(c *fiber.Ctx, err error) error {
|
||||
return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
|
||||
"error": err.Error(),
|
||||
})
|
||||
},
|
||||
})
|
||||
|
||||
app.Use(recover.New())
|
||||
app.Use(cors.New(cors.Config{
|
||||
AllowOrigins: "*",
|
||||
AllowHeaders: "Origin, Content-Type, Accept, Authorization",
|
||||
AllowMethods: "GET, POST, PUT, DELETE, OPTIONS",
|
||||
}))
|
||||
app.Use(middleware.Logging(middleware.LoggingConfig{}))
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{
|
||||
"status": "ok",
|
||||
"service": "computer-svc",
|
||||
"models": registry.Count(),
|
||||
})
|
||||
})
|
||||
|
||||
api := app.Group("/api/v1/computer")
|
||||
|
||||
api.Post("/execute", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Query string `json:"query"`
|
||||
UserID string `json:"userId"`
|
||||
Options computer.ExecuteOptions `json:"options"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
|
||||
}
|
||||
|
||||
if req.Query == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "query is required"})
|
||||
}
|
||||
|
||||
if req.UserID == "" || req.UserID == "anonymous" {
|
||||
req.UserID = "00000000-0000-0000-0000-000000000000"
|
||||
}
|
||||
|
||||
task, err := comp.Execute(c.Context(), req.UserID, req.Query, req.Options)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(task)
|
||||
})
|
||||
|
||||
api.Get("/tasks", func(c *fiber.Ctx) error {
|
||||
userID := c.Query("userId", "")
|
||||
limit := c.QueryInt("limit", 20)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
if userID == "" || userID == "anonymous" {
|
||||
userID = "00000000-0000-0000-0000-000000000000"
|
||||
}
|
||||
|
||||
tasks, err := comp.GetUserTasks(c.Context(), userID, limit, offset)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"tasks": tasks,
|
||||
"count": len(tasks),
|
||||
})
|
||||
})
|
||||
|
||||
api.Get("/tasks/:id", func(c *fiber.Ctx) error {
|
||||
taskID := c.Params("id")
|
||||
|
||||
task, err := comp.GetStatus(c.Context(), taskID)
|
||||
if err != nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "task not found"})
|
||||
}
|
||||
|
||||
return c.JSON(task)
|
||||
})
|
||||
|
||||
api.Get("/tasks/:id/stream", func(c *fiber.Ctx) error {
|
||||
taskID := c.Params("id")
|
||||
|
||||
c.Set("Content-Type", "text/event-stream")
|
||||
c.Set("Cache-Control", "no-cache")
|
||||
c.Set("Connection", "keep-alive")
|
||||
c.Set("Transfer-Encoding", "chunked")
|
||||
|
||||
eventCh, err := comp.Stream(c.Context(), taskID)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
c.Context().SetBodyStreamWriter(func(w *bufio.Writer) {
|
||||
for event := range eventCh {
|
||||
data, _ := json.Marshal(event)
|
||||
fmt.Fprintf(w, "data: %s\n\n", data)
|
||||
w.Flush()
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
api.Post("/tasks/:id/resume", func(c *fiber.Ctx) error {
|
||||
taskID := c.Params("id")
|
||||
|
||||
var req struct {
|
||||
UserInput string `json:"userInput"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
|
||||
}
|
||||
|
||||
if err := comp.Resume(c.Context(), taskID, req.UserInput); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"status": "resumed"})
|
||||
})
|
||||
|
||||
api.Delete("/tasks/:id", func(c *fiber.Ctx) error {
|
||||
taskID := c.Params("id")
|
||||
|
||||
if err := comp.Cancel(c.Context(), taskID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"status": "cancelled"})
|
||||
})
|
||||
|
||||
api.Get("/tasks/:id/artifacts", func(c *fiber.Ctx) error {
|
||||
taskID := c.Params("id")
|
||||
|
||||
artifacts, err := artifactRepo.GetByTaskID(c.Context(), taskID)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"artifacts": artifacts,
|
||||
"count": len(artifacts),
|
||||
})
|
||||
})
|
||||
|
||||
api.Get("/artifacts/:id", func(c *fiber.Ctx) error {
|
||||
artifactID := c.Params("id")
|
||||
|
||||
artifact, err := artifactRepo.GetByID(c.Context(), artifactID)
|
||||
if err != nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "artifact not found"})
|
||||
}
|
||||
|
||||
return c.JSON(artifact)
|
||||
})
|
||||
|
||||
api.Get("/artifacts/:id/download", func(c *fiber.Ctx) error {
|
||||
artifactID := c.Params("id")
|
||||
|
||||
artifact, err := artifactRepo.GetByID(c.Context(), artifactID)
|
||||
if err != nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "artifact not found"})
|
||||
}
|
||||
|
||||
if artifact.MimeType != "" {
|
||||
c.Set("Content-Type", artifact.MimeType)
|
||||
} else {
|
||||
c.Set("Content-Type", "application/octet-stream")
|
||||
}
|
||||
c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", artifact.Name))
|
||||
|
||||
return c.Send(artifact.Content)
|
||||
})
|
||||
|
||||
api.Get("/models", func(c *fiber.Ctx) error {
|
||||
models := registry.GetAll()
|
||||
return c.JSON(fiber.Map{
|
||||
"models": models,
|
||||
"count": len(models),
|
||||
})
|
||||
})
|
||||
|
||||
api.Get("/connectors", func(c *fiber.Ctx) error {
|
||||
info := connectorHub.GetInfo()
|
||||
return c.JSON(fiber.Map{
|
||||
"connectors": info,
|
||||
"count": len(info),
|
||||
})
|
||||
})
|
||||
|
||||
api.Post("/connectors/:id/execute", func(c *fiber.Ctx) error {
|
||||
connectorID := c.Params("id")
|
||||
|
||||
var req struct {
|
||||
Action string `json:"action"`
|
||||
Params map[string]interface{} `json:"params"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "invalid request body"})
|
||||
}
|
||||
|
||||
result, err := connectorHub.Execute(c.Context(), connectorID, req.Action, req.Params)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
port := getEnv("COMPUTER_SVC_PORT", "3030")
|
||||
addr := ":" + port
|
||||
|
||||
go func() {
|
||||
log.Printf("Computer service starting on %s", addr)
|
||||
if err := app.Listen(addr); err != nil {
|
||||
log.Fatalf("Failed to start server: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-quit
|
||||
|
||||
log.Println("Shutting down...")
|
||||
comp.StopScheduler()
|
||||
app.Shutdown()
|
||||
}
|
||||
|
||||
func setupModels(registry *llm.ModelRegistry, cfg *config.Config) {
|
||||
// Timeweb Cloud AI (приоритетный провайдер для России)
|
||||
if cfg.TimewebAgentAccessID != "" && cfg.TimewebAPIKey != "" {
|
||||
timewebClient, err := llm.NewTimewebClient(llm.TimewebConfig{
|
||||
ProviderID: "timeweb",
|
||||
ModelKey: "gpt-4o",
|
||||
BaseURL: cfg.TimewebAPIBaseURL,
|
||||
AgentAccessID: cfg.TimewebAgentAccessID,
|
||||
APIKey: cfg.TimewebAPIKey,
|
||||
ProxySource: cfg.TimewebProxySource,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "timeweb-gpt-4o",
|
||||
Provider: "timeweb",
|
||||
Model: "gpt-4o",
|
||||
Capabilities: []llm.ModelCapability{llm.CapSearch, llm.CapFast, llm.CapVision, llm.CapCoding, llm.CapCreative, llm.CapReasoning},
|
||||
CostPer1K: 0.005,
|
||||
MaxContext: 128000,
|
||||
MaxTokens: 16384,
|
||||
Priority: 0,
|
||||
Description: "GPT-4o via Timeweb Cloud AI",
|
||||
}, timewebClient)
|
||||
log.Println("Timeweb GPT-4o registered")
|
||||
} else {
|
||||
log.Printf("Failed to create Timeweb client: %v", err)
|
||||
}
|
||||
|
||||
timewebMiniClient, err := llm.NewTimewebClient(llm.TimewebConfig{
|
||||
ProviderID: "timeweb",
|
||||
ModelKey: "gpt-4o-mini",
|
||||
BaseURL: cfg.TimewebAPIBaseURL,
|
||||
AgentAccessID: cfg.TimewebAgentAccessID,
|
||||
APIKey: cfg.TimewebAPIKey,
|
||||
ProxySource: cfg.TimewebProxySource,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "timeweb-gpt-4o-mini",
|
||||
Provider: "timeweb",
|
||||
Model: "gpt-4o-mini",
|
||||
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapCoding},
|
||||
CostPer1K: 0.00015,
|
||||
MaxContext: 128000,
|
||||
MaxTokens: 16384,
|
||||
Priority: 0,
|
||||
Description: "GPT-4o-mini via Timeweb Cloud AI",
|
||||
}, timewebMiniClient)
|
||||
log.Println("Timeweb GPT-4o-mini registered")
|
||||
}
|
||||
}
|
||||
|
||||
// OpenAI прямой (fallback если Timeweb недоступен)
|
||||
if cfg.OpenAIAPIKey != "" {
|
||||
openaiClient, err := llm.NewOpenAIClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
ModelKey: "gpt-4o",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "gpt-4o",
|
||||
Provider: "openai",
|
||||
Model: "gpt-4o",
|
||||
Capabilities: []llm.ModelCapability{llm.CapSearch, llm.CapFast, llm.CapVision, llm.CapCoding, llm.CapCreative},
|
||||
CostPer1K: 0.005,
|
||||
MaxContext: 128000,
|
||||
MaxTokens: 16384,
|
||||
Priority: 10,
|
||||
}, openaiClient)
|
||||
}
|
||||
|
||||
miniClient, err := llm.NewOpenAIClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
ModelKey: "gpt-4o-mini",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "gpt-4o-mini",
|
||||
Provider: "openai",
|
||||
Model: "gpt-4o-mini",
|
||||
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapCoding},
|
||||
CostPer1K: 0.00015,
|
||||
MaxContext: 128000,
|
||||
MaxTokens: 16384,
|
||||
Priority: 10,
|
||||
}, miniClient)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.AnthropicAPIKey != "" {
|
||||
opusClient, err := llm.NewAnthropicClient(llm.ProviderConfig{
|
||||
ProviderID: "anthropic",
|
||||
ModelKey: "claude-3-opus-20240229",
|
||||
APIKey: cfg.AnthropicAPIKey,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "claude-3-opus",
|
||||
Provider: "anthropic",
|
||||
Model: "claude-3-opus-20240229",
|
||||
Capabilities: []llm.ModelCapability{llm.CapReasoning, llm.CapCoding, llm.CapCreative, llm.CapLongContext},
|
||||
CostPer1K: 0.015,
|
||||
MaxContext: 200000,
|
||||
MaxTokens: 4096,
|
||||
Priority: 1,
|
||||
}, opusClient)
|
||||
}
|
||||
|
||||
sonnetClient, err := llm.NewAnthropicClient(llm.ProviderConfig{
|
||||
ProviderID: "anthropic",
|
||||
ModelKey: "claude-3-5-sonnet-20241022",
|
||||
APIKey: cfg.AnthropicAPIKey,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "claude-3-sonnet",
|
||||
Provider: "anthropic",
|
||||
Model: "claude-3-5-sonnet-20241022",
|
||||
Capabilities: []llm.ModelCapability{llm.CapCoding, llm.CapCreative, llm.CapFast},
|
||||
CostPer1K: 0.003,
|
||||
MaxContext: 200000,
|
||||
MaxTokens: 8192,
|
||||
Priority: 1,
|
||||
}, sonnetClient)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.GeminiAPIKey != "" {
|
||||
geminiClient, err := llm.NewGeminiClient(llm.ProviderConfig{
|
||||
ProviderID: "gemini",
|
||||
ModelKey: "gemini-1.5-pro",
|
||||
APIKey: cfg.GeminiAPIKey,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "gemini-1.5-pro",
|
||||
Provider: "gemini",
|
||||
Model: "gemini-1.5-pro",
|
||||
Capabilities: []llm.ModelCapability{llm.CapLongContext, llm.CapSearch, llm.CapVision, llm.CapMath},
|
||||
CostPer1K: 0.00125,
|
||||
MaxContext: 2000000,
|
||||
MaxTokens: 8192,
|
||||
Priority: 1,
|
||||
}, geminiClient)
|
||||
}
|
||||
|
||||
flashClient, err := llm.NewGeminiClient(llm.ProviderConfig{
|
||||
ProviderID: "gemini",
|
||||
ModelKey: "gemini-1.5-flash",
|
||||
APIKey: cfg.GeminiAPIKey,
|
||||
})
|
||||
if err == nil {
|
||||
registry.Register(llm.ModelSpec{
|
||||
ID: "gemini-1.5-flash",
|
||||
Provider: "gemini",
|
||||
Model: "gemini-1.5-flash",
|
||||
Capabilities: []llm.ModelCapability{llm.CapFast, llm.CapVision},
|
||||
CostPer1K: 0.000075,
|
||||
MaxContext: 1000000,
|
||||
MaxTokens: 8192,
|
||||
Priority: 2,
|
||||
}, flashClient)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Registered %d models", registry.Count())
|
||||
}
|
||||
|
||||
func setupConnectors(hub *connectors.ConnectorHub, cfg *config.Config) {
|
||||
if smtpHost := getEnv("SMTP_HOST", ""); smtpHost != "" {
|
||||
emailConn := connectors.NewEmailConnector(connectors.EmailConfig{
|
||||
SMTPHost: smtpHost,
|
||||
SMTPPort: getEnvInt("SMTP_PORT", 587),
|
||||
Username: getEnv("SMTP_USERNAME", ""),
|
||||
Password: getEnv("SMTP_PASSWORD", ""),
|
||||
FromAddress: getEnv("SMTP_FROM", ""),
|
||||
FromName: getEnv("SMTP_FROM_NAME", "GooSeek Computer"),
|
||||
UseTLS: true,
|
||||
AllowHTML: true,
|
||||
})
|
||||
hub.Register(emailConn)
|
||||
log.Println("Email connector registered")
|
||||
}
|
||||
|
||||
if botToken := getEnv("TELEGRAM_BOT_TOKEN", ""); botToken != "" {
|
||||
tgConn := connectors.NewTelegramConnector(connectors.TelegramConfig{
|
||||
BotToken: botToken,
|
||||
})
|
||||
hub.Register(tgConn)
|
||||
log.Println("Telegram connector registered")
|
||||
}
|
||||
|
||||
webhookConn := connectors.NewWebhookConnector(connectors.WebhookConfig{
|
||||
Timeout: 30 * time.Second,
|
||||
MaxRetries: 3,
|
||||
})
|
||||
hub.Register(webhookConn)
|
||||
log.Println("Webhook connector registered")
|
||||
|
||||
if s3Endpoint := getEnv("S3_ENDPOINT", ""); s3Endpoint != "" {
|
||||
storageConn, err := connectors.NewStorageConnector(connectors.StorageConfig{
|
||||
Endpoint: s3Endpoint,
|
||||
AccessKeyID: getEnv("S3_ACCESS_KEY", ""),
|
||||
SecretAccessKey: getEnv("S3_SECRET_KEY", ""),
|
||||
BucketName: getEnv("S3_BUCKET", "gooseek-artifacts"),
|
||||
UseSSL: getEnv("S3_USE_SSL", "true") == "true",
|
||||
Region: getEnv("S3_REGION", "us-east-1"),
|
||||
PublicURL: getEnv("S3_PUBLIC_URL", ""),
|
||||
})
|
||||
if err == nil {
|
||||
hub.Register(storageConn)
|
||||
log.Println("Storage connector registered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func getEnv(key, defaultValue string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
var i int
|
||||
fmt.Sscanf(value, "%d", &i)
|
||||
return i
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
511
backend/cmd/discover-svc/main.go
Normal file
511
backend/cmd/discover-svc/main.go
Normal file
@@ -0,0 +1,511 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/db"
|
||||
"github.com/gooseek/backend/internal/search"
|
||||
"github.com/gooseek/backend/pkg/cache"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
)
|
||||
|
||||
type DigestCitation struct {
|
||||
Index int `json:"index"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Domain string `json:"domain"`
|
||||
}
|
||||
|
||||
type Digest struct {
|
||||
Topic string `json:"topic"`
|
||||
Region string `json:"region"`
|
||||
ClusterTitle string `json:"clusterTitle"`
|
||||
SummaryRu string `json:"summaryRu"`
|
||||
Citations []DigestCitation `json:"citations"`
|
||||
SourcesCount int `json:"sourcesCount"`
|
||||
FollowUp []string `json:"followUp"`
|
||||
Thumbnail string `json:"thumbnail"`
|
||||
ShortDescription string `json:"shortDescription"`
|
||||
MainURL string `json:"mainUrl"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type ArticleSummary struct {
|
||||
URL string `json:"url"`
|
||||
Events []string `json:"events"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
}
|
||||
|
||||
type DiscoverStore struct {
|
||||
digests map[string]*Digest
|
||||
articleSummaries map[string]*ArticleSummary
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func NewDiscoverStore() *DiscoverStore {
|
||||
return &DiscoverStore{
|
||||
digests: make(map[string]*Digest),
|
||||
articleSummaries: make(map[string]*ArticleSummary),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) GetDigest(topic, region, title string) *Digest {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
key := fmt.Sprintf("%s:%s:%s", topic, region, title)
|
||||
return s.digests[key]
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) GetDigestByURL(url string) *Digest {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
normalizedURL := normalizeURL(url)
|
||||
for _, d := range s.digests {
|
||||
if normalizeURL(d.MainURL) == normalizedURL {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) GetDigests(topic, region string) []*Digest {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
result := make([]*Digest, 0)
|
||||
prefix := fmt.Sprintf("%s:%s:", topic, region)
|
||||
for k, d := range s.digests {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
result = append(result, d)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) UpsertDigest(d *Digest) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
key := fmt.Sprintf("%s:%s:%s", d.Topic, d.Region, d.ClusterTitle)
|
||||
d.CreatedAt = time.Now()
|
||||
s.digests[key] = d
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) DeleteDigests(topic, region string) int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
count := 0
|
||||
prefix := fmt.Sprintf("%s:%s:", topic, region)
|
||||
for k := range s.digests {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
delete(s.digests, k)
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) GetArticleSummary(url string) *ArticleSummary {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
key := articleSummaryKey(url)
|
||||
return s.articleSummaries[key]
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) SaveArticleSummary(url string, events []string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
key := articleSummaryKey(url)
|
||||
s.articleSummaries[key] = &ArticleSummary{
|
||||
URL: url,
|
||||
Events: events,
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *DiscoverStore) DeleteArticleSummary(url string) bool {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
key := articleSummaryKey(url)
|
||||
if _, ok := s.articleSummaries[key]; ok {
|
||||
delete(s.articleSummaries, key)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func articleSummaryKey(url string) string {
|
||||
hash := sha256.Sum256([]byte(normalizeURL(url)))
|
||||
return hex.EncodeToString(hash[:16])
|
||||
}
|
||||
|
||||
func normalizeURL(url string) string {
|
||||
url = strings.TrimSpace(url)
|
||||
url = strings.TrimSuffix(url, "/")
|
||||
url = strings.TrimPrefix(url, "https://")
|
||||
url = strings.TrimPrefix(url, "http://")
|
||||
url = strings.TrimPrefix(url, "www.")
|
||||
return url
|
||||
}
|
||||
|
||||
func extractDomain(url string) string {
|
||||
normalized := normalizeURL(url)
|
||||
if idx := strings.Index(normalized, "/"); idx > 0 {
|
||||
return normalized[:idx]
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
store := NewDiscoverStore()
|
||||
searchClient := search.NewSearXNGClient(cfg)
|
||||
|
||||
var database *db.PostgresDB
|
||||
var digestRepo *db.DigestRepository
|
||||
var summaryRepo *db.ArticleSummaryRepository
|
||||
|
||||
if cfg.DatabaseURL != "" {
|
||||
database, err = db.NewPostgresDB(cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Printf("PostgreSQL unavailable: %v (falling back to in-memory)", err)
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if err := database.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Migration warning: %v", err)
|
||||
}
|
||||
cancel()
|
||||
defer database.Close()
|
||||
|
||||
digestRepo = db.NewDigestRepository(database)
|
||||
summaryRepo = db.NewArticleSummaryRepository(database)
|
||||
log.Println("PostgreSQL connected")
|
||||
}
|
||||
}
|
||||
|
||||
var redisCache *cache.RedisCache
|
||||
if cfg.RedisURL != "" {
|
||||
redisCache, err = cache.NewRedisCache(cfg.RedisURL, "gooseek:discover")
|
||||
if err != nil {
|
||||
log.Printf("Redis cache unavailable: %v (falling back to in-memory)", err)
|
||||
} else {
|
||||
log.Printf("Redis cache connected")
|
||||
defer redisCache.Close()
|
||||
}
|
||||
}
|
||||
|
||||
_ = digestRepo
|
||||
_ = summaryRepo
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 100 * 1024 * 1024,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/ready", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ready"})
|
||||
})
|
||||
|
||||
app.Get("/metrics", func(c *fiber.Ctx) error {
|
||||
c.Set("Content-Type", "text/plain; charset=utf-8")
|
||||
return c.SendString(
|
||||
"# HELP gooseek_up Service is up (1) or down (0)\n" +
|
||||
"# TYPE gooseek_up gauge\n" +
|
||||
"gooseek_up 1\n",
|
||||
)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/discover/digest", func(c *fiber.Ctx) error {
|
||||
url := c.Query("url")
|
||||
if url != "" {
|
||||
digest := store.GetDigestByURL(url)
|
||||
if digest == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"message": "digest not found"})
|
||||
}
|
||||
return c.JSON(digest)
|
||||
}
|
||||
|
||||
topic := c.Query("topic")
|
||||
region := c.Query("region")
|
||||
title := c.Query("title")
|
||||
|
||||
if topic == "" || region == "" || title == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"message": "topic, region, title (or url) required"})
|
||||
}
|
||||
|
||||
digest := store.GetDigest(topic, region, title)
|
||||
if digest == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"message": "digest not found"})
|
||||
}
|
||||
|
||||
return c.JSON(digest)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/discover/digest", func(c *fiber.Ctx) error {
|
||||
var d Digest
|
||||
if err := c.BodyParser(&d); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if d.Topic == "" || d.Region == "" || d.ClusterTitle == "" || d.SummaryRu == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"message": "topic, region, clusterTitle, summaryRu required"})
|
||||
}
|
||||
|
||||
store.UpsertDigest(&d)
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
app.Delete("/api/v1/discover/digest", func(c *fiber.Ctx) error {
|
||||
topic := c.Query("topic")
|
||||
region := c.Query("region")
|
||||
|
||||
if topic == "" || region == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"message": "topic, region required"})
|
||||
}
|
||||
|
||||
deleted := store.DeleteDigests(topic, region)
|
||||
return c.JSON(fiber.Map{"deleted": deleted})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/discover/article-summary", func(c *fiber.Ctx) error {
|
||||
url := c.Query("url")
|
||||
if url == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"message": "url required"})
|
||||
}
|
||||
|
||||
if redisCache != nil {
|
||||
events, err := redisCache.GetCachedArticleSummary(c.Context(), url)
|
||||
if err == nil && len(events) > 0 {
|
||||
return c.JSON(fiber.Map{"events": events})
|
||||
}
|
||||
}
|
||||
|
||||
if summaryRepo != nil {
|
||||
summary, err := summaryRepo.GetByURL(c.Context(), url)
|
||||
if err == nil && summary != nil {
|
||||
if redisCache != nil {
|
||||
redisCache.CacheArticleSummary(c.Context(), url, summary.Events, 24*time.Hour)
|
||||
}
|
||||
return c.JSON(fiber.Map{"events": summary.Events})
|
||||
}
|
||||
}
|
||||
|
||||
summary := store.GetArticleSummary(url)
|
||||
if summary == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"message": "not found"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"events": summary.Events})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/discover/article-summary", func(c *fiber.Ctx) error {
|
||||
var body struct {
|
||||
URL string `json:"url"`
|
||||
Events []string `json:"events"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&body); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if body.URL == "" || len(body.Events) == 0 {
|
||||
return c.Status(400).JSON(fiber.Map{"message": "url and events[] required"})
|
||||
}
|
||||
|
||||
store.SaveArticleSummary(body.URL, body.Events)
|
||||
|
||||
if summaryRepo != nil {
|
||||
ttl := 7 * 24 * time.Hour
|
||||
if err := summaryRepo.Save(c.Context(), body.URL, body.Events, ttl); err != nil {
|
||||
log.Printf("postgres save article-summary error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if redisCache != nil {
|
||||
ttl := 24 * time.Hour
|
||||
if err := redisCache.CacheArticleSummary(c.Context(), body.URL, body.Events, ttl); err != nil {
|
||||
log.Printf("redis cache article-summary error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("article-summary saved: %s (%d events)", body.URL[:min(60, len(body.URL))], len(body.Events))
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
app.Delete("/api/v1/discover/article-summary", func(c *fiber.Ctx) error {
|
||||
url := c.Query("url")
|
||||
if url == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"message": "url required"})
|
||||
}
|
||||
|
||||
deleted := store.DeleteArticleSummary(url)
|
||||
log.Printf("article-summary deleted: %s (deleted=%v)", url[:min(60, len(url))], deleted)
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/discover/search", func(c *fiber.Ctx) error {
|
||||
q := c.Query("q")
|
||||
if q == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"message": "Query q is required"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout)
|
||||
defer cancel()
|
||||
|
||||
result, err := searchClient.Search(ctx, q, &search.SearchOptions{PageNo: 1})
|
||||
if err != nil {
|
||||
return c.Status(503).JSON(fiber.Map{"message": "Search failed"})
|
||||
}
|
||||
|
||||
if len(result.Results) > 10 {
|
||||
result.Results = result.Results[:10]
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"results": result.Results})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/discover", func(c *fiber.Ctx) error {
|
||||
topic := c.Query("topic", "tech")
|
||||
region := c.Query("region", "world")
|
||||
|
||||
digests := store.GetDigests(topic, region)
|
||||
if len(digests) > 0 {
|
||||
blogs := make([]fiber.Map, len(digests))
|
||||
for i, d := range digests {
|
||||
content := d.ShortDescription
|
||||
if content == "" && len(d.SummaryRu) > 200 {
|
||||
content = d.SummaryRu[:200] + "…"
|
||||
} else if content == "" {
|
||||
content = d.SummaryRu
|
||||
}
|
||||
|
||||
blogs[i] = fiber.Map{
|
||||
"title": d.ClusterTitle,
|
||||
"content": content,
|
||||
"url": d.MainURL,
|
||||
"thumbnail": d.Thumbnail,
|
||||
"sourcesCount": d.SourcesCount,
|
||||
"digestId": fmt.Sprintf("%s:%s:%s", d.Topic, d.Region, d.ClusterTitle),
|
||||
}
|
||||
}
|
||||
return c.JSON(fiber.Map{"blogs": blogs})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout*2)
|
||||
defer cancel()
|
||||
|
||||
queries := getQueriesForTopic(topic, region)
|
||||
results, err := searchClient.Search(ctx, queries[0], &search.SearchOptions{
|
||||
Categories: []string{"news"},
|
||||
PageNo: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(503).JSON(fiber.Map{"message": "Search failed"})
|
||||
}
|
||||
|
||||
blogs := make([]fiber.Map, 0, 7)
|
||||
for i, r := range results.Results {
|
||||
if i >= 7 {
|
||||
break
|
||||
}
|
||||
thumbnail := r.Thumbnail
|
||||
if thumbnail == "" {
|
||||
thumbnail = r.ThumbnailSrc
|
||||
}
|
||||
if thumbnail == "" {
|
||||
thumbnail = r.ImgSrc
|
||||
}
|
||||
|
||||
content := r.Content
|
||||
if content == "" {
|
||||
content = r.Title
|
||||
}
|
||||
if len(content) > 300 {
|
||||
content = content[:300] + "…"
|
||||
}
|
||||
|
||||
blogs = append(blogs, fiber.Map{
|
||||
"title": r.Title,
|
||||
"content": content,
|
||||
"url": r.URL,
|
||||
"thumbnail": thumbnail,
|
||||
})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"blogs": blogs})
|
||||
})
|
||||
|
||||
port := getEnvInt("DISCOVER_SVC_PORT", 3002)
|
||||
log.Printf("discover-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getQueriesForTopic(topic, region string) []string {
|
||||
queries := map[string]map[string][]string{
|
||||
"tech": {
|
||||
"world": {"technology news AI innovation"},
|
||||
"russia": {"технологии новости IT инновации"},
|
||||
"eu": {"technology news Europe AI"},
|
||||
},
|
||||
"finance": {
|
||||
"world": {"finance news economy markets"},
|
||||
"russia": {"финансы новости экономика рынки"},
|
||||
"eu": {"finance news Europe economy"},
|
||||
},
|
||||
"sports": {
|
||||
"world": {"sports news football Olympics"},
|
||||
"russia": {"спорт новости футбол хоккей"},
|
||||
"eu": {"sports news football Champions League"},
|
||||
},
|
||||
}
|
||||
|
||||
if topicQueries, ok := queries[topic]; ok {
|
||||
if regionQueries, ok := topicQueries[region]; ok {
|
||||
return regionQueries
|
||||
}
|
||||
if defaultQueries, ok := topicQueries["world"]; ok {
|
||||
return defaultQueries
|
||||
}
|
||||
}
|
||||
|
||||
return []string{"news today"}
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
307
backend/cmd/file-svc/main.go
Normal file
307
backend/cmd/file-svc/main.go
Normal file
@@ -0,0 +1,307 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/db"
|
||||
"github.com/gooseek/backend/internal/files"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/gooseek/backend/pkg/middleware"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
var database *db.PostgresDB
|
||||
var fileRepo *db.FileRepository
|
||||
|
||||
if cfg.DatabaseURL != "" {
|
||||
database, err = db.NewPostgresDB(cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Printf("Database unavailable: %v", err)
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if err := database.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Migration warning: %v", err)
|
||||
}
|
||||
cancel()
|
||||
defer database.Close()
|
||||
|
||||
fileRepo = db.NewFileRepository(database)
|
||||
log.Println("PostgreSQL connected")
|
||||
}
|
||||
}
|
||||
|
||||
var llmClient llm.Client
|
||||
if cfg.OpenAIAPIKey != "" {
|
||||
llmClient, err = llm.NewClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
ModelKey: "gpt-4o",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to create OpenAI client: %v", err)
|
||||
}
|
||||
} else if cfg.AnthropicAPIKey != "" {
|
||||
llmClient, err = llm.NewClient(llm.ProviderConfig{
|
||||
ProviderID: "anthropic",
|
||||
ModelKey: "claude-3-5-sonnet-20241022",
|
||||
APIKey: cfg.AnthropicAPIKey,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to create Anthropic client: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
storagePath := os.Getenv("FILE_STORAGE_PATH")
|
||||
if storagePath == "" {
|
||||
storagePath = "/tmp/gooseek-files"
|
||||
}
|
||||
|
||||
var fileAnalyzer *files.FileAnalyzer
|
||||
if llmClient != nil {
|
||||
fileAnalyzer = files.NewFileAnalyzer(llmClient, storagePath)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 100 * 1024 * 1024,
|
||||
ReadTimeout: 60 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
if cfg.JWTSecret != "" || cfg.AuthSvcURL != "" {
|
||||
app.Use(middleware.JWT(middleware.JWTConfig{
|
||||
Secret: cfg.JWTSecret,
|
||||
AuthSvcURL: cfg.AuthSvcURL,
|
||||
AllowGuest: false,
|
||||
SkipPaths: []string{"/health", "/ready"},
|
||||
}))
|
||||
}
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/ready", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ready"})
|
||||
})
|
||||
|
||||
api := app.Group("/api/v1/files")
|
||||
|
||||
api.Post("/upload", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil || fileAnalyzer == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Unauthorized"})
|
||||
}
|
||||
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "No file uploaded"})
|
||||
}
|
||||
|
||||
if file.Size > 50*1024*1024 {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "File too large (max 50MB)"})
|
||||
}
|
||||
|
||||
f, err := file.Open()
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to read file"})
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
storagePath, fileSize, err := fileAnalyzer.SaveFile(file.Filename, f)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to save file"})
|
||||
}
|
||||
|
||||
buf := make([]byte, 512)
|
||||
f.Seek(0, 0)
|
||||
f.Read(buf)
|
||||
mimeType := files.DetectMimeType(file.Filename, buf)
|
||||
|
||||
uploadedFile := &db.UploadedFile{
|
||||
UserID: userID,
|
||||
Filename: file.Filename,
|
||||
FileType: mimeType,
|
||||
FileSize: fileSize,
|
||||
StoragePath: storagePath,
|
||||
Metadata: map[string]interface{}{},
|
||||
}
|
||||
|
||||
if err := fileRepo.Create(c.Context(), uploadedFile); err != nil {
|
||||
fileAnalyzer.DeleteFile(storagePath)
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to save file record"})
|
||||
}
|
||||
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
result, err := fileAnalyzer.AnalyzeFile(ctx, storagePath, mimeType)
|
||||
if err != nil {
|
||||
log.Printf("File analysis failed for %s: %v", uploadedFile.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
fileRepo.UpdateExtractedText(ctx, uploadedFile.ID, result.ExtractedText)
|
||||
}()
|
||||
|
||||
return c.Status(201).JSON(fiber.Map{
|
||||
"id": uploadedFile.ID,
|
||||
"filename": uploadedFile.Filename,
|
||||
"fileType": uploadedFile.FileType,
|
||||
"fileSize": uploadedFile.FileSize,
|
||||
"status": "processing",
|
||||
})
|
||||
})
|
||||
|
||||
api.Get("/", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Unauthorized"})
|
||||
}
|
||||
|
||||
limit := c.QueryInt("limit", 50)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
files, err := fileRepo.GetByUserID(c.Context(), userID, limit, offset)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get files"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"files": files})
|
||||
})
|
||||
|
||||
api.Get("/:id", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
return c.JSON(file)
|
||||
})
|
||||
|
||||
api.Get("/:id/content", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"id": file.ID,
|
||||
"filename": file.Filename,
|
||||
"extractedText": file.ExtractedText,
|
||||
})
|
||||
})
|
||||
|
||||
api.Post("/:id/analyze", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil || fileAnalyzer == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
result, err := fileAnalyzer.AnalyzeFile(c.Context(), file.StoragePath, file.FileType)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Analysis failed: " + err.Error()})
|
||||
}
|
||||
|
||||
fileRepo.UpdateExtractedText(c.Context(), fileID, result.ExtractedText)
|
||||
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
api.Delete("/:id", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil || fileAnalyzer == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
fileAnalyzer.DeleteFile(file.StoragePath)
|
||||
|
||||
if err := fileRepo.Delete(c.Context(), fileID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to delete file"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
port := getEnvInt("FILE_SVC_PORT", 3026)
|
||||
log.Printf("file-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
139
backend/cmd/finance-heatmap-svc/main.go
Normal file
139
backend/cmd/finance-heatmap-svc/main.go
Normal file
@@ -0,0 +1,139 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/finance"
|
||||
)
|
||||
|
||||
func main() {
|
||||
heatmapSvc := finance.NewHeatmapService(finance.HeatmapConfig{
|
||||
CacheTTL: 5 * time.Minute,
|
||||
RefreshInterval: time.Minute,
|
||||
})
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/heatmap/:market", func(c *fiber.Ctx) error {
|
||||
market := c.Params("market")
|
||||
timeRange := c.Query("range", "1d")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
heatmap, err := heatmapSvc.GetMarketHeatmap(ctx, market, timeRange)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(heatmap)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/heatmap/:market/treemap", func(c *fiber.Ctx) error {
|
||||
market := c.Params("market")
|
||||
timeRange := c.Query("range", "1d")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
heatmap, err := heatmapSvc.GetMarketHeatmap(ctx, market, timeRange)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
treemapData := heatmapSvc.GenerateTreemapData(heatmap)
|
||||
return c.JSON(treemapData)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/heatmap/:market/grid", func(c *fiber.Ctx) error {
|
||||
market := c.Params("market")
|
||||
timeRange := c.Query("range", "1d")
|
||||
rows := c.QueryInt("rows", 5)
|
||||
cols := c.QueryInt("cols", 10)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
heatmap, err := heatmapSvc.GetMarketHeatmap(ctx, market, timeRange)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
gridData := heatmapSvc.GenerateGridData(heatmap, rows, cols)
|
||||
return c.JSON(fiber.Map{"grid": gridData, "rows": rows, "cols": cols})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/heatmap/:market/sector/:sector", func(c *fiber.Ctx) error {
|
||||
market := c.Params("market")
|
||||
sector := c.Params("sector")
|
||||
timeRange := c.Query("range", "1d")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
heatmap, err := heatmapSvc.GetSectorHeatmap(ctx, market, sector, timeRange)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(heatmap)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/movers/:market", func(c *fiber.Ctx) error {
|
||||
market := c.Params("market")
|
||||
count := c.QueryInt("count", 10)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
movers, err := heatmapSvc.GetTopMovers(ctx, market, count)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(movers)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/markets", func(c *fiber.Ctx) error {
|
||||
markets := []map[string]interface{}{
|
||||
{"id": "sp500", "name": "S&P 500", "region": "us"},
|
||||
{"id": "nasdaq", "name": "NASDAQ", "region": "us"},
|
||||
{"id": "dow", "name": "Dow Jones", "region": "us"},
|
||||
{"id": "moex", "name": "MOEX", "region": "ru"},
|
||||
{"id": "crypto", "name": "Cryptocurrency", "region": "global"},
|
||||
{"id": "forex", "name": "Forex", "region": "global"},
|
||||
}
|
||||
return c.JSON(fiber.Map{"markets": markets})
|
||||
})
|
||||
|
||||
port := getEnvInt("PORT", 3033)
|
||||
log.Printf("finance-heatmap-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
553
backend/cmd/labs-svc/main.go
Normal file
553
backend/cmd/labs-svc/main.go
Normal file
@@ -0,0 +1,553 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/labs"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type ReportStore struct {
|
||||
reports map[string]*labs.Report
|
||||
}
|
||||
|
||||
func NewReportStore() *ReportStore {
|
||||
return &ReportStore{
|
||||
reports: make(map[string]*labs.Report),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ReportStore) Save(report *labs.Report) {
|
||||
s.reports[report.ID] = report
|
||||
}
|
||||
|
||||
func (s *ReportStore) Get(id string) *labs.Report {
|
||||
return s.reports[id]
|
||||
}
|
||||
|
||||
func (s *ReportStore) List(limit, offset int) []*labs.Report {
|
||||
result := make([]*labs.Report, 0)
|
||||
i := 0
|
||||
for _, r := range s.reports {
|
||||
if i >= offset && len(result) < limit {
|
||||
result = append(result, r)
|
||||
}
|
||||
i++
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *ReportStore) Delete(id string) bool {
|
||||
if _, ok := s.reports[id]; ok {
|
||||
delete(s.reports, id)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
var llmClient llm.Client
|
||||
if cfg.OpenAIAPIKey != "" {
|
||||
client, err := llm.NewOpenAIClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
ModelKey: "gpt-4o-mini",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create OpenAI client:", err)
|
||||
}
|
||||
llmClient = client
|
||||
} else if cfg.AnthropicAPIKey != "" {
|
||||
client, err := llm.NewAnthropicClient(llm.ProviderConfig{
|
||||
ProviderID: "anthropic",
|
||||
APIKey: cfg.AnthropicAPIKey,
|
||||
ModelKey: "claude-3-5-sonnet-20241022",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create Anthropic client:", err)
|
||||
}
|
||||
llmClient = client
|
||||
}
|
||||
|
||||
generator := labs.NewGenerator(llmClient)
|
||||
store := NewReportStore()
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 100 * 1024 * 1024,
|
||||
ReadTimeout: 60 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/generate", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Query string `json:"query"`
|
||||
Data interface{} `json:"data"`
|
||||
Theme string `json:"theme,omitempty"`
|
||||
Locale string `json:"locale,omitempty"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
report, err := generator.GenerateReport(ctx, labs.GenerateOptions{
|
||||
Query: req.Query,
|
||||
Data: req.Data,
|
||||
Theme: req.Theme,
|
||||
Locale: req.Locale,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
store.Save(report)
|
||||
|
||||
return c.JSON(report)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/visualize", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Data interface{} `json:"data"`
|
||||
Config interface{} `json:"config,omitempty"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := createVisualizationFromRequest(generator, req.Type, req.Title, req.Data, req.Config)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/chart", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
Labels []string `json:"labels"`
|
||||
Datasets []labs.ChartDataset `json:"datasets"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
var viz labs.Visualization
|
||||
switch req.Type {
|
||||
case "bar", "bar_chart":
|
||||
if len(req.Datasets) > 0 {
|
||||
viz = generator.CreateBarChart(req.Title, req.Labels, req.Datasets[0].Data)
|
||||
}
|
||||
case "line", "line_chart":
|
||||
viz = generator.CreateLineChart(req.Title, req.Labels, req.Datasets)
|
||||
case "pie", "pie_chart":
|
||||
if len(req.Datasets) > 0 {
|
||||
viz = generator.CreatePieChart(req.Title, req.Labels, req.Datasets[0].Data)
|
||||
}
|
||||
default:
|
||||
viz = generator.CreateBarChart(req.Title, req.Labels, req.Datasets[0].Data)
|
||||
}
|
||||
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/table", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
Columns []labs.TableColumn `json:"columns"`
|
||||
Rows []labs.TableRow `json:"rows"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateTable(req.Title, req.Columns, req.Rows)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/stat-cards", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
Cards []labs.StatCard `json:"cards"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateStatCards(req.Title, req.Cards)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/kpi", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
Value interface{} `json:"value"`
|
||||
Change float64 `json:"change"`
|
||||
Unit string `json:"unit"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateKPI(req.Title, req.Value, req.Change, req.Unit)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/heatmap", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
XLabels []string `json:"xLabels"`
|
||||
YLabels []string `json:"yLabels"`
|
||||
Values [][]float64 `json:"values"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateHeatmap(req.Title, req.XLabels, req.YLabels, req.Values)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/code", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
Code string `json:"code"`
|
||||
Language string `json:"language"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateCodeBlock(req.Title, req.Code, req.Language)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/markdown", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateMarkdown(req.Title, req.Content)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/tabs", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
Tabs []labs.TabItem `json:"tabs"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateTabs(req.Title, req.Tabs)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/accordion", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
Items []labs.AccordionItem `json:"items"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
viz := generator.CreateAccordion(req.Title, req.Items)
|
||||
return c.JSON(viz)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/labs/reports", func(c *fiber.Ctx) error {
|
||||
limit := c.QueryInt("limit", 20)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
reports := store.List(limit, offset)
|
||||
return c.JSON(fiber.Map{"reports": reports, "count": len(reports)})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/labs/reports/:id", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
report := store.Get(id)
|
||||
if report == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Report not found"})
|
||||
}
|
||||
return c.JSON(report)
|
||||
})
|
||||
|
||||
app.Delete("/api/v1/labs/reports/:id", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
if store.Delete(id) {
|
||||
return c.JSON(fiber.Map{"success": true})
|
||||
}
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Report not found"})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/labs/reports/:id/export", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
format := c.Query("format", "html")
|
||||
|
||||
report := store.Get(id)
|
||||
if report == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Report not found"})
|
||||
}
|
||||
|
||||
switch format {
|
||||
case "html":
|
||||
html := exportToHTML(report)
|
||||
c.Set("Content-Type", "text/html")
|
||||
c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.html\"", report.ID))
|
||||
return c.SendString(html)
|
||||
case "json":
|
||||
c.Set("Content-Type", "application/json")
|
||||
c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.json\"", report.ID))
|
||||
return c.JSON(report)
|
||||
default:
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Unsupported format"})
|
||||
}
|
||||
})
|
||||
|
||||
port := getEnvInt("LABS_SVC_PORT", 3031)
|
||||
log.Printf("labs-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func createVisualizationFromRequest(g *labs.Generator, vizType, title string, data, config interface{}) labs.Visualization {
|
||||
switch vizType {
|
||||
case "bar_chart":
|
||||
return parseChartRequest(g, labs.VizBarChart, title, data)
|
||||
case "line_chart":
|
||||
return parseChartRequest(g, labs.VizLineChart, title, data)
|
||||
case "pie_chart":
|
||||
return parseChartRequest(g, labs.VizPieChart, title, data)
|
||||
case "table":
|
||||
return parseTableRequest(g, title, data)
|
||||
case "stat_cards":
|
||||
return parseStatCardsRequest(g, title, data)
|
||||
case "kpi":
|
||||
return parseKPIRequest(g, title, data)
|
||||
case "markdown":
|
||||
content := ""
|
||||
if dataMap, ok := data.(map[string]interface{}); ok {
|
||||
content, _ = dataMap["content"].(string)
|
||||
}
|
||||
return g.CreateMarkdown(title, content)
|
||||
case "code_block":
|
||||
code, lang := "", ""
|
||||
if dataMap, ok := data.(map[string]interface{}); ok {
|
||||
code, _ = dataMap["code"].(string)
|
||||
lang, _ = dataMap["language"].(string)
|
||||
}
|
||||
return g.CreateCodeBlock(title, code, lang)
|
||||
default:
|
||||
return g.CreateMarkdown(title, fmt.Sprintf("%v", data))
|
||||
}
|
||||
}
|
||||
|
||||
func parseChartRequest(g *labs.Generator, vizType labs.VisualizationType, title string, data interface{}) labs.Visualization {
|
||||
dataMap, ok := data.(map[string]interface{})
|
||||
if !ok {
|
||||
return g.CreateMarkdown(title, "Invalid chart data")
|
||||
}
|
||||
|
||||
labels := make([]string, 0)
|
||||
if labelsRaw, ok := dataMap["labels"].([]interface{}); ok {
|
||||
for _, l := range labelsRaw {
|
||||
labels = append(labels, fmt.Sprintf("%v", l))
|
||||
}
|
||||
}
|
||||
|
||||
values := make([]float64, 0)
|
||||
if valuesRaw, ok := dataMap["values"].([]interface{}); ok {
|
||||
for _, v := range valuesRaw {
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
values = append(values, val)
|
||||
case int:
|
||||
values = append(values, float64(val))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch vizType {
|
||||
case labs.VizBarChart:
|
||||
return g.CreateBarChart(title, labels, values)
|
||||
case labs.VizPieChart:
|
||||
return g.CreatePieChart(title, labels, values)
|
||||
default:
|
||||
return g.CreateLineChart(title, labels, []labs.ChartDataset{{Label: title, Data: values}})
|
||||
}
|
||||
}
|
||||
|
||||
func parseTableRequest(g *labs.Generator, title string, data interface{}) labs.Visualization {
|
||||
dataMap, ok := data.(map[string]interface{})
|
||||
if !ok {
|
||||
return g.CreateMarkdown(title, "Invalid table data")
|
||||
}
|
||||
|
||||
columns := make([]labs.TableColumn, 0)
|
||||
if colsRaw, ok := dataMap["columns"].([]interface{}); ok {
|
||||
for _, c := range colsRaw {
|
||||
if colMap, ok := c.(map[string]interface{}); ok {
|
||||
col := labs.TableColumn{}
|
||||
if v, ok := colMap["key"].(string); ok {
|
||||
col.Key = v
|
||||
}
|
||||
if v, ok := colMap["label"].(string); ok {
|
||||
col.Label = v
|
||||
}
|
||||
columns = append(columns, col)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rows := make([]labs.TableRow, 0)
|
||||
if rowsRaw, ok := dataMap["rows"].([]interface{}); ok {
|
||||
for _, r := range rowsRaw {
|
||||
if rowMap, ok := r.(map[string]interface{}); ok {
|
||||
rows = append(rows, labs.TableRow(rowMap))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return g.CreateTable(title, columns, rows)
|
||||
}
|
||||
|
||||
func parseStatCardsRequest(g *labs.Generator, title string, data interface{}) labs.Visualization {
|
||||
dataMap, ok := data.(map[string]interface{})
|
||||
if !ok {
|
||||
return g.CreateMarkdown(title, "Invalid stat cards data")
|
||||
}
|
||||
|
||||
cards := make([]labs.StatCard, 0)
|
||||
if cardsRaw, ok := dataMap["cards"].([]interface{}); ok {
|
||||
for _, c := range cardsRaw {
|
||||
if cardMap, ok := c.(map[string]interface{}); ok {
|
||||
card := labs.StatCard{ID: uuid.New().String()}
|
||||
if v, ok := cardMap["title"].(string); ok {
|
||||
card.Title = v
|
||||
}
|
||||
if v, ok := cardMap["value"]; ok {
|
||||
card.Value = v
|
||||
}
|
||||
if v, ok := cardMap["change"].(float64); ok {
|
||||
card.Change = v
|
||||
}
|
||||
if v, ok := cardMap["color"].(string); ok {
|
||||
card.Color = v
|
||||
}
|
||||
cards = append(cards, card)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return g.CreateStatCards(title, cards)
|
||||
}
|
||||
|
||||
func parseKPIRequest(g *labs.Generator, title string, data interface{}) labs.Visualization {
|
||||
dataMap, ok := data.(map[string]interface{})
|
||||
if !ok {
|
||||
return g.CreateKPI(title, data, 0, "")
|
||||
}
|
||||
|
||||
value := dataMap["value"]
|
||||
change := 0.0
|
||||
if v, ok := dataMap["change"].(float64); ok {
|
||||
change = v
|
||||
}
|
||||
unit := ""
|
||||
if v, ok := dataMap["unit"].(string); ok {
|
||||
unit = v
|
||||
}
|
||||
|
||||
return g.CreateKPI(title, value, change, unit)
|
||||
}
|
||||
|
||||
func exportToHTML(report *labs.Report) string {
|
||||
sectionsHTML := ""
|
||||
for _, section := range report.Sections {
|
||||
vizHTML := ""
|
||||
for _, viz := range section.Visualizations {
|
||||
dataJSON, _ := json.Marshal(viz.Data)
|
||||
vizHTML += fmt.Sprintf(`
|
||||
<div class="visualization" data-type="%s">
|
||||
<h3>%s</h3>
|
||||
<div class="viz-container" data-config='%s'></div>
|
||||
</div>
|
||||
`, viz.Type, viz.Title, string(dataJSON))
|
||||
}
|
||||
sectionsHTML += fmt.Sprintf(`
|
||||
<section>
|
||||
<h2>%s</h2>
|
||||
%s
|
||||
</section>
|
||||
`, section.Title, vizHTML)
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>%s</title>
|
||||
<style>
|
||||
body { font-family: system-ui, sans-serif; max-width: 1200px; margin: 0 auto; padding: 20px; }
|
||||
h1 { color: #1f2937; }
|
||||
h2 { color: #374151; border-bottom: 2px solid #e5e7eb; padding-bottom: 10px; }
|
||||
.visualization { margin: 20px 0; padding: 20px; background: #f9fafb; border-radius: 8px; }
|
||||
.viz-container { min-height: 200px; }
|
||||
</style>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<h1>%s</h1>
|
||||
%s
|
||||
</body>
|
||||
</html>`, report.Title, report.Title, sectionsHTML)
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
404
backend/cmd/learning-svc/main.go
Normal file
404
backend/cmd/learning-svc/main.go
Normal file
@@ -0,0 +1,404 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/learning"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
)
|
||||
|
||||
type LessonStore struct {
|
||||
lessons map[string]*learning.StepByStepLesson
|
||||
}
|
||||
|
||||
func NewLessonStore() *LessonStore {
|
||||
return &LessonStore{
|
||||
lessons: make(map[string]*learning.StepByStepLesson),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *LessonStore) Save(lesson *learning.StepByStepLesson) {
|
||||
s.lessons[lesson.ID] = lesson
|
||||
}
|
||||
|
||||
func (s *LessonStore) Get(id string) *learning.StepByStepLesson {
|
||||
return s.lessons[id]
|
||||
}
|
||||
|
||||
func (s *LessonStore) List(limit, offset int) []*learning.StepByStepLesson {
|
||||
result := make([]*learning.StepByStepLesson, 0)
|
||||
i := 0
|
||||
for _, l := range s.lessons {
|
||||
if i >= offset && len(result) < limit {
|
||||
result = append(result, l)
|
||||
}
|
||||
i++
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *LessonStore) Delete(id string) bool {
|
||||
if _, ok := s.lessons[id]; ok {
|
||||
delete(s.lessons, id)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
var llmClient llm.Client
|
||||
|
||||
// Priority 1: Timeweb Cloud AI (recommended for production)
|
||||
if cfg.TimewebAgentAccessID != "" && cfg.TimewebAPIKey != "" {
|
||||
client, err := llm.NewTimewebClient(llm.TimewebConfig{
|
||||
ProviderID: "timeweb",
|
||||
BaseURL: cfg.TimewebAPIBaseURL,
|
||||
AgentAccessID: cfg.TimewebAgentAccessID,
|
||||
APIKey: cfg.TimewebAPIKey,
|
||||
ModelKey: cfg.DefaultLLMModel,
|
||||
ProxySource: cfg.TimewebProxySource,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Warning: Failed to create Timeweb client: %v", err)
|
||||
} else {
|
||||
llmClient = client
|
||||
log.Println("Using Timeweb Cloud AI as LLM provider")
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 2: Anthropic
|
||||
if llmClient == nil && cfg.AnthropicAPIKey != "" && !isJWT(cfg.AnthropicAPIKey) {
|
||||
client, err := llm.NewAnthropicClient(llm.ProviderConfig{
|
||||
ProviderID: "anthropic",
|
||||
APIKey: cfg.AnthropicAPIKey,
|
||||
ModelKey: "claude-3-5-sonnet-20241022",
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Warning: Failed to create Anthropic client: %v", err)
|
||||
} else {
|
||||
llmClient = client
|
||||
log.Println("Using Anthropic as LLM provider")
|
||||
}
|
||||
}
|
||||
|
||||
// Priority 3: OpenAI (only if it's a real OpenAI key, not Timeweb JWT)
|
||||
if llmClient == nil && cfg.OpenAIAPIKey != "" && !isJWT(cfg.OpenAIAPIKey) {
|
||||
client, err := llm.NewOpenAIClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
ModelKey: "gpt-4o-mini",
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Warning: Failed to create OpenAI client: %v", err)
|
||||
} else {
|
||||
llmClient = client
|
||||
log.Println("Using OpenAI as LLM provider")
|
||||
}
|
||||
}
|
||||
|
||||
if llmClient == nil {
|
||||
log.Fatal("No LLM provider configured. Please set TIMEWEB_AGENT_ACCESS_ID + TIMEWEB_API_KEY, or OPENAI_API_KEY, or ANTHROPIC_API_KEY")
|
||||
}
|
||||
|
||||
generator := learning.NewLearningGenerator(llmClient)
|
||||
store := NewLessonStore()
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 50 * 1024 * 1024,
|
||||
ReadTimeout: 120 * time.Second,
|
||||
WriteTimeout: 120 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/learning/lesson", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Topic string `json:"topic"`
|
||||
Query string `json:"query"`
|
||||
Difficulty string `json:"difficulty"`
|
||||
Mode string `json:"mode"`
|
||||
MaxSteps int `json:"maxSteps"`
|
||||
Locale string `json:"locale"`
|
||||
IncludeCode bool `json:"includeCode"`
|
||||
IncludeQuiz bool `json:"includeQuiz"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
difficulty := learning.DifficultyBeginner
|
||||
switch req.Difficulty {
|
||||
case "intermediate":
|
||||
difficulty = learning.DifficultyIntermediate
|
||||
case "advanced":
|
||||
difficulty = learning.DifficultyAdvanced
|
||||
case "expert":
|
||||
difficulty = learning.DifficultyExpert
|
||||
}
|
||||
|
||||
mode := learning.ModeExplain
|
||||
switch req.Mode {
|
||||
case "guided":
|
||||
mode = learning.ModeGuided
|
||||
case "interactive":
|
||||
mode = learning.ModeInteractive
|
||||
case "practice":
|
||||
mode = learning.ModePractice
|
||||
case "quiz":
|
||||
mode = learning.ModeQuiz
|
||||
}
|
||||
|
||||
lesson, err := generator.GenerateLesson(ctx, learning.GenerateLessonOptions{
|
||||
Topic: req.Topic,
|
||||
Query: req.Query,
|
||||
Difficulty: difficulty,
|
||||
Mode: mode,
|
||||
MaxSteps: req.MaxSteps,
|
||||
Locale: req.Locale,
|
||||
IncludeCode: req.IncludeCode,
|
||||
IncludeQuiz: req.IncludeQuiz,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
store.Save(lesson)
|
||||
|
||||
return c.JSON(lesson)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/learning/explain", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Topic string `json:"topic"`
|
||||
Difficulty string `json:"difficulty"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
difficulty := learning.DifficultyBeginner
|
||||
switch req.Difficulty {
|
||||
case "intermediate":
|
||||
difficulty = learning.DifficultyIntermediate
|
||||
case "advanced":
|
||||
difficulty = learning.DifficultyAdvanced
|
||||
case "expert":
|
||||
difficulty = learning.DifficultyExpert
|
||||
}
|
||||
|
||||
step, err := generator.GenerateExplanation(ctx, req.Topic, difficulty, req.Locale)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(step)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/learning/quiz", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Topic string `json:"topic"`
|
||||
NumQuestions int `json:"numQuestions"`
|
||||
Difficulty string `json:"difficulty"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
if req.NumQuestions == 0 {
|
||||
req.NumQuestions = 5
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
difficulty := learning.DifficultyBeginner
|
||||
switch req.Difficulty {
|
||||
case "intermediate":
|
||||
difficulty = learning.DifficultyIntermediate
|
||||
case "advanced":
|
||||
difficulty = learning.DifficultyAdvanced
|
||||
case "expert":
|
||||
difficulty = learning.DifficultyExpert
|
||||
}
|
||||
|
||||
questions, err := generator.GenerateQuiz(ctx, req.Topic, req.NumQuestions, difficulty, req.Locale)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"questions": questions})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/learning/practice", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Topic string `json:"topic"`
|
||||
Language string `json:"language"`
|
||||
Difficulty string `json:"difficulty"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
difficulty := learning.DifficultyBeginner
|
||||
switch req.Difficulty {
|
||||
case "intermediate":
|
||||
difficulty = learning.DifficultyIntermediate
|
||||
case "advanced":
|
||||
difficulty = learning.DifficultyAdvanced
|
||||
case "expert":
|
||||
difficulty = learning.DifficultyExpert
|
||||
}
|
||||
|
||||
exercise, err := generator.GeneratePracticeExercise(ctx, req.Topic, req.Language, difficulty, req.Locale)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(exercise)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/learning/lessons", func(c *fiber.Ctx) error {
|
||||
limit := c.QueryInt("limit", 20)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
lessons := store.List(limit, offset)
|
||||
|
||||
summaries := make([]map[string]interface{}, 0)
|
||||
for _, l := range lessons {
|
||||
summaries = append(summaries, map[string]interface{}{
|
||||
"id": l.ID,
|
||||
"title": l.Title,
|
||||
"topic": l.Topic,
|
||||
"difficulty": l.Difficulty,
|
||||
"mode": l.Mode,
|
||||
"stepsCount": len(l.Steps),
|
||||
"estimatedTime": l.EstimatedTime,
|
||||
"progress": l.Progress,
|
||||
"createdAt": l.CreatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"lessons": summaries, "count": len(summaries)})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/learning/lessons/:id", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
lesson := store.Get(id)
|
||||
if lesson == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Lesson not found"})
|
||||
}
|
||||
return c.JSON(lesson)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/learning/lessons/:id/complete-step", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
lesson := store.Get(id)
|
||||
if lesson == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Lesson not found"})
|
||||
}
|
||||
|
||||
var req struct {
|
||||
StepIndex int `json:"stepIndex"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
lesson.CompleteStep(req.StepIndex)
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"success": true,
|
||||
"progress": lesson.Progress,
|
||||
})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/learning/lessons/:id/submit-answer", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
lesson := store.Get(id)
|
||||
if lesson == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Lesson not found"})
|
||||
}
|
||||
|
||||
var req struct {
|
||||
StepIndex int `json:"stepIndex"`
|
||||
SelectedOptions []string `json:"selectedOptions"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
correct, explanation := lesson.SubmitQuizAnswer(req.StepIndex, req.SelectedOptions)
|
||||
|
||||
if correct {
|
||||
lesson.CompleteStep(req.StepIndex)
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"correct": correct,
|
||||
"explanation": explanation,
|
||||
"progress": lesson.Progress,
|
||||
})
|
||||
})
|
||||
|
||||
app.Delete("/api/v1/learning/lessons/:id", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
if store.Delete(id) {
|
||||
return c.JSON(fiber.Map{"success": true})
|
||||
}
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Lesson not found"})
|
||||
})
|
||||
|
||||
port := getEnvInt("LEARNING_SVC_PORT", 3034)
|
||||
log.Printf("learning-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
func isJWT(s string) bool {
|
||||
return len(s) > 10 && s[:3] == "eyJ"
|
||||
}
|
||||
191
backend/cmd/llm-svc/main.go
Normal file
191
backend/cmd/llm-svc/main.go
Normal file
@@ -0,0 +1,191 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/gooseek/backend/pkg/ndjson"
|
||||
)
|
||||
|
||||
type GenerateRequest struct {
|
||||
ProviderID string `json:"providerId"`
|
||||
ModelKey string `json:"key"`
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
Options struct {
|
||||
MaxTokens int `json:"maxTokens"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
Stream bool `json:"stream"`
|
||||
} `json:"options"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
StreamRequestBody: true,
|
||||
BodyLimit: 10 * 1024 * 1024,
|
||||
ReadTimeout: time.Minute,
|
||||
WriteTimeout: 5 * time.Minute,
|
||||
IdleTimeout: 2 * time.Minute,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/providers", func(c *fiber.Ctx) error {
|
||||
providers := []fiber.Map{}
|
||||
|
||||
if cfg.OpenAIAPIKey != "" {
|
||||
providers = append(providers, fiber.Map{
|
||||
"id": "openai",
|
||||
"name": "OpenAI",
|
||||
"models": []string{"gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "gpt-3.5-turbo"},
|
||||
})
|
||||
}
|
||||
|
||||
if cfg.AnthropicAPIKey != "" {
|
||||
providers = append(providers, fiber.Map{
|
||||
"id": "anthropic",
|
||||
"name": "Anthropic",
|
||||
"models": []string{"claude-3-5-sonnet-20241022", "claude-3-opus-20240229", "claude-3-haiku-20240307"},
|
||||
})
|
||||
}
|
||||
|
||||
if cfg.GeminiAPIKey != "" {
|
||||
providers = append(providers, fiber.Map{
|
||||
"id": "gemini",
|
||||
"name": "Google Gemini",
|
||||
"models": []string{"gemini-1.5-pro", "gemini-1.5-flash", "gemini-2.0-flash-exp"},
|
||||
})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"providers": providers,
|
||||
"envOnlyMode": true,
|
||||
})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/providers/ui-config", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{
|
||||
"sections": []interface{}{},
|
||||
})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/generate", func(c *fiber.Ctx) error {
|
||||
var req GenerateRequest
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if len(req.Messages) == 0 {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Messages required"})
|
||||
}
|
||||
|
||||
client, err := llm.NewClient(llm.ProviderConfig{
|
||||
ProviderID: req.ProviderID,
|
||||
ModelKey: req.ModelKey,
|
||||
APIKey: getAPIKey(cfg, req.ProviderID),
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
messages := make([]llm.Message, len(req.Messages))
|
||||
for i, m := range req.Messages {
|
||||
messages[i] = llm.Message{
|
||||
Role: llm.Role(m.Role),
|
||||
Content: m.Content,
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.LLMTimeout)
|
||||
defer cancel()
|
||||
|
||||
if req.Options.Stream {
|
||||
stream, err := client.StreamText(ctx, llm.StreamRequest{
|
||||
Messages: messages,
|
||||
Options: llm.StreamOptions{
|
||||
MaxTokens: req.Options.MaxTokens,
|
||||
Temperature: req.Options.Temperature,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
c.Set("Content-Type", "application/x-ndjson")
|
||||
c.Set("Cache-Control", "no-cache")
|
||||
|
||||
c.Context().SetBodyStreamWriter(func(w *bufio.Writer) {
|
||||
writer := ndjson.NewWriter(w)
|
||||
for chunk := range stream {
|
||||
writer.Write(fiber.Map{
|
||||
"type": "chunk",
|
||||
"chunk": chunk.ContentChunk,
|
||||
})
|
||||
w.Flush()
|
||||
}
|
||||
writer.Write(fiber.Map{"type": "done"})
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
response, err := client.GenerateText(ctx, llm.StreamRequest{
|
||||
Messages: messages,
|
||||
Options: llm.StreamOptions{
|
||||
MaxTokens: req.Options.MaxTokens,
|
||||
Temperature: req.Options.Temperature,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"content": response,
|
||||
})
|
||||
})
|
||||
|
||||
port := cfg.LLMSvcPort
|
||||
log.Printf("llm-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getAPIKey(cfg *config.Config, providerID string) string {
|
||||
switch providerID {
|
||||
case "openai", "timeweb":
|
||||
return cfg.OpenAIAPIKey
|
||||
case "anthropic":
|
||||
return cfg.AnthropicAPIKey
|
||||
case "gemini", "google":
|
||||
return cfg.GeminiAPIKey
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
if os.Getenv("PORT") == "" {
|
||||
os.Setenv("PORT", "3020")
|
||||
}
|
||||
}
|
||||
424
backend/cmd/podcast-svc/main.go
Normal file
424
backend/cmd/podcast-svc/main.go
Normal file
@@ -0,0 +1,424 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/internal/podcast"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
)
|
||||
|
||||
type PodcastStore struct {
|
||||
podcasts map[string]*podcast.Podcast
|
||||
}
|
||||
|
||||
func NewPodcastStore() *PodcastStore {
|
||||
return &PodcastStore{
|
||||
podcasts: make(map[string]*podcast.Podcast),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *PodcastStore) Save(p *podcast.Podcast) {
|
||||
s.podcasts[p.ID] = p
|
||||
}
|
||||
|
||||
func (s *PodcastStore) Get(id string) *podcast.Podcast {
|
||||
return s.podcasts[id]
|
||||
}
|
||||
|
||||
func (s *PodcastStore) List(limit, offset int) []*podcast.Podcast {
|
||||
result := make([]*podcast.Podcast, 0)
|
||||
i := 0
|
||||
for _, p := range s.podcasts {
|
||||
if i >= offset && len(result) < limit {
|
||||
result = append(result, p)
|
||||
}
|
||||
i++
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *PodcastStore) GetLatest() *podcast.Podcast {
|
||||
var latest *podcast.Podcast
|
||||
for _, p := range s.podcasts {
|
||||
if latest == nil || p.GeneratedAt.After(latest.GeneratedAt) {
|
||||
latest = p
|
||||
}
|
||||
}
|
||||
return latest
|
||||
}
|
||||
|
||||
func (s *PodcastStore) GetByDate(date time.Time) *podcast.Podcast {
|
||||
dateStr := date.Format("2006-01-02")
|
||||
for _, p := range s.podcasts {
|
||||
if p.Date.Format("2006-01-02") == dateStr {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
var llmClient llm.Client
|
||||
if cfg.OpenAIAPIKey != "" {
|
||||
client, err := llm.NewOpenAIClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
ModelKey: "gpt-4o-mini",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create OpenAI client:", err)
|
||||
}
|
||||
llmClient = client
|
||||
} else if cfg.AnthropicAPIKey != "" {
|
||||
client, err := llm.NewAnthropicClient(llm.ProviderConfig{
|
||||
ProviderID: "anthropic",
|
||||
APIKey: cfg.AnthropicAPIKey,
|
||||
ModelKey: "claude-3-5-sonnet-20241022",
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal("Failed to create Anthropic client:", err)
|
||||
}
|
||||
llmClient = client
|
||||
}
|
||||
|
||||
var ttsClient podcast.TTSClient
|
||||
elevenLabsKey := os.Getenv("ELEVENLABS_API_KEY")
|
||||
if elevenLabsKey != "" {
|
||||
ttsClient = podcast.NewElevenLabsTTS(elevenLabsKey)
|
||||
} else {
|
||||
ttsClient = &podcast.DummyTTS{}
|
||||
}
|
||||
|
||||
generator := podcast.NewPodcastGenerator(llmClient, ttsClient, podcast.GeneratorConfig{
|
||||
DefaultDuration: 300,
|
||||
MaxDuration: 1800,
|
||||
OutputDir: "/data/podcasts",
|
||||
})
|
||||
|
||||
store := NewPodcastStore()
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 50 * 1024 * 1024,
|
||||
ReadTimeout: 120 * time.Second,
|
||||
WriteTimeout: 120 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/podcast/generate", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Type string `json:"type"`
|
||||
Topics []string `json:"topics"`
|
||||
NewsItems []podcast.NewsItem `json:"newsItems"`
|
||||
Duration int `json:"duration"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
podcastType := podcast.PodcastDaily
|
||||
switch req.Type {
|
||||
case "weekly":
|
||||
podcastType = podcast.PodcastWeekly
|
||||
case "topic_deep":
|
||||
podcastType = podcast.PodcastTopicDeep
|
||||
case "breaking":
|
||||
podcastType = podcast.PodcastBreaking
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
p, err := generator.GenerateDailyPodcast(ctx, podcast.GenerateOptions{
|
||||
Type: podcastType,
|
||||
Topics: req.Topics,
|
||||
NewsItems: req.NewsItems,
|
||||
Duration: req.Duration,
|
||||
Locale: req.Locale,
|
||||
IncludeIntro: true,
|
||||
IncludeOutro: true,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
store.Save(p)
|
||||
|
||||
return c.JSON(p)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/podcast/generate-daily", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
NewsItems []podcast.NewsItem `json:"newsItems"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
if req.Locale == "" {
|
||||
req.Locale = "ru"
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
|
||||
p, err := generator.GenerateDailyPodcast(ctx, podcast.GenerateOptions{
|
||||
Type: podcast.PodcastDaily,
|
||||
NewsItems: req.NewsItems,
|
||||
Date: time.Now(),
|
||||
Duration: 300,
|
||||
Locale: req.Locale,
|
||||
IncludeIntro: true,
|
||||
IncludeOutro: true,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
store.Save(p)
|
||||
|
||||
return c.JSON(p)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/podcast/generate-weekly", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
NewsItems []podcast.NewsItem `json:"newsItems"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
|
||||
p, err := generator.GenerateWeeklySummary(ctx, req.NewsItems, req.Locale)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
store.Save(p)
|
||||
|
||||
return c.JSON(p)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/podcast/generate-topic", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
Topic string `json:"topic"`
|
||||
Articles []podcast.NewsItem `json:"articles"`
|
||||
Locale string `json:"locale"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second)
|
||||
defer cancel()
|
||||
|
||||
p, err := generator.GenerateTopicDeepDive(ctx, req.Topic, req.Articles, req.Locale)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
store.Save(p)
|
||||
|
||||
return c.JSON(p)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/podcast/:id/audio", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
p := store.Get(id)
|
||||
if p == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Podcast not found"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second)
|
||||
defer cancel()
|
||||
|
||||
audioData, err := generator.GenerateAudio(ctx, p)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
c.Set("Content-Type", "audio/mpeg")
|
||||
c.Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.mp3\"", p.ID))
|
||||
return c.Send(audioData)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/podcasts", func(c *fiber.Ctx) error {
|
||||
limit := c.QueryInt("limit", 20)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
podcasts := store.List(limit, offset)
|
||||
|
||||
summaries := make([]map[string]interface{}, 0)
|
||||
for _, p := range podcasts {
|
||||
summaries = append(summaries, map[string]interface{}{
|
||||
"id": p.ID,
|
||||
"title": p.Title,
|
||||
"description": p.Description,
|
||||
"type": p.Type,
|
||||
"date": p.Date,
|
||||
"duration": p.Duration,
|
||||
"status": p.Status,
|
||||
"topics": p.Topics,
|
||||
"audioUrl": p.AudioURL,
|
||||
"generatedAt": p.GeneratedAt,
|
||||
})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"podcasts": summaries, "count": len(summaries)})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/podcasts/latest", func(c *fiber.Ctx) error {
|
||||
p := store.GetLatest()
|
||||
if p == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "No podcasts found"})
|
||||
}
|
||||
return c.JSON(p)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/podcasts/today", func(c *fiber.Ctx) error {
|
||||
p := store.GetByDate(time.Now())
|
||||
if p == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "No podcast for today"})
|
||||
}
|
||||
return c.JSON(p)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/podcasts/:id", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
p := store.Get(id)
|
||||
if p == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Podcast not found"})
|
||||
}
|
||||
return c.JSON(p)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/podcasts/:id/transcript", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
p := store.Get(id)
|
||||
if p == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Podcast not found"})
|
||||
}
|
||||
return c.JSON(fiber.Map{
|
||||
"id": p.ID,
|
||||
"title": p.Title,
|
||||
"transcript": p.Transcript,
|
||||
"segments": p.Segments,
|
||||
})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/podcasts/:id/segments", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
p := store.Get(id)
|
||||
if p == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Podcast not found"})
|
||||
}
|
||||
return c.JSON(fiber.Map{
|
||||
"segments": p.Segments,
|
||||
})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/podcast/:id/publish", func(c *fiber.Ctx) error {
|
||||
id := c.Params("id")
|
||||
p := store.Get(id)
|
||||
if p == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Podcast not found"})
|
||||
}
|
||||
|
||||
if p.Status != podcast.StatusReady {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Podcast audio not ready"})
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
p.Status = podcast.StatusPublished
|
||||
p.PublishedAt = &now
|
||||
|
||||
return c.JSON(fiber.Map{"success": true, "publishedAt": now})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/podcast/rss", func(c *fiber.Ctx) error {
|
||||
podcasts := store.List(50, 0)
|
||||
|
||||
rss := generateRSSFeed(podcasts, c.BaseURL())
|
||||
|
||||
c.Set("Content-Type", "application/rss+xml")
|
||||
return c.SendString(rss)
|
||||
})
|
||||
|
||||
port := getEnvInt("PODCAST_SVC_PORT", 3032)
|
||||
log.Printf("podcast-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func generateRSSFeed(podcasts []*podcast.Podcast, baseURL string) string {
|
||||
items := ""
|
||||
for _, p := range podcasts {
|
||||
if p.Status == podcast.StatusPublished && p.AudioURL != "" {
|
||||
pubDate := ""
|
||||
if p.PublishedAt != nil {
|
||||
pubDate = p.PublishedAt.Format(time.RFC1123Z)
|
||||
}
|
||||
items += fmt.Sprintf(`
|
||||
<item>
|
||||
<title>%s</title>
|
||||
<description><![CDATA[%s]]></description>
|
||||
<pubDate>%s</pubDate>
|
||||
<enclosure url="%s" type="audio/mpeg" length="0"/>
|
||||
<guid>%s</guid>
|
||||
<itunes:duration>%d</itunes:duration>
|
||||
</item>`, p.Title, p.Description, pubDate, p.AudioURL, p.ID, p.Duration)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0"
|
||||
xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd"
|
||||
xmlns:atom="http://www.w3.org/2005/Atom">
|
||||
<channel>
|
||||
<title>GooSeek Daily</title>
|
||||
<link>%s</link>
|
||||
<description>Ежедневный подкаст с главными новостями от GooSeek</description>
|
||||
<language>ru</language>
|
||||
<itunes:author>GooSeek</itunes:author>
|
||||
<itunes:category text="News"/>
|
||||
<atom:link href="%s/api/v1/podcast/rss" rel="self" type="application/rss+xml"/>
|
||||
%s
|
||||
</channel>
|
||||
</rss>`, baseURL, baseURL, items)
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
284
backend/cmd/scraper-svc/main.go
Normal file
284
backend/cmd/scraper-svc/main.go
Normal file
@@ -0,0 +1,284 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
)
|
||||
|
||||
type ScrapeRequest struct {
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type ScrapeResponse struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 10 * 1024 * 1024,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Post("/api/v1/scrape", func(c *fiber.Ctx) error {
|
||||
var req ScrapeRequest
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if req.URL == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "URL is required"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.ScrapeTimeout)
|
||||
defer cancel()
|
||||
|
||||
result := scrapeURL(ctx, req.URL, cfg)
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
app.Post("/api/v1/scrape/batch", func(c *fiber.Ctx) error {
|
||||
var req struct {
|
||||
URLs []string `json:"urls"`
|
||||
}
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request body"})
|
||||
}
|
||||
|
||||
if len(req.URLs) == 0 {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "URLs are required"})
|
||||
}
|
||||
|
||||
if len(req.URLs) > 10 {
|
||||
req.URLs = req.URLs[:10]
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.ScrapeTimeout*3)
|
||||
defer cancel()
|
||||
|
||||
results := make([]ScrapeResponse, len(req.URLs))
|
||||
resultCh := make(chan struct {
|
||||
index int
|
||||
result ScrapeResponse
|
||||
}, len(req.URLs))
|
||||
|
||||
for i, url := range req.URLs {
|
||||
go func(idx int, u string) {
|
||||
resultCh <- struct {
|
||||
index int
|
||||
result ScrapeResponse
|
||||
}{idx, scrapeURL(ctx, u, cfg)}
|
||||
}(i, url)
|
||||
}
|
||||
|
||||
for range req.URLs {
|
||||
r := <-resultCh
|
||||
results[r.index] = r.result
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"results": results})
|
||||
})
|
||||
|
||||
port := cfg.ScraperSvcPort
|
||||
log.Printf("scraper-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func scrapeURL(ctx context.Context, url string, cfg *config.Config) ScrapeResponse {
|
||||
if cfg.Crawl4AIURL != "" {
|
||||
result, err := scrapeWithCrawl4AI(ctx, url, cfg.Crawl4AIURL)
|
||||
if err == nil && result.Success {
|
||||
return *result
|
||||
}
|
||||
}
|
||||
|
||||
return scrapeDirectly(ctx, url)
|
||||
}
|
||||
|
||||
func scrapeWithCrawl4AI(ctx context.Context, url, crawl4aiURL string) (*ScrapeResponse, error) {
|
||||
reqBody := fmt.Sprintf(`{
|
||||
"urls": ["%s"],
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"cache_mode": "default",
|
||||
"page_timeout": 20000
|
||||
}
|
||||
}
|
||||
}`, url)
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", crawl4aiURL+"/crawl", strings.NewReader(reqBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{Timeout: 25 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("Crawl4AI returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
markdown := extractMarkdown(string(body))
|
||||
title := extractTitle(string(body))
|
||||
|
||||
if len(markdown) > 100 {
|
||||
return &ScrapeResponse{
|
||||
URL: url,
|
||||
Title: title,
|
||||
Content: truncate(markdown, 15000),
|
||||
Success: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("insufficient content from Crawl4AI")
|
||||
}
|
||||
|
||||
func scrapeDirectly(ctx context.Context, url string) ScrapeResponse {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return ScrapeResponse{URL: url, Success: false, Error: err.Error()}
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "GooSeek-Scraper/1.0")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return ScrapeResponse{URL: url, Success: false, Error: err.Error()}
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ScrapeResponse{URL: url, Success: false, Error: fmt.Sprintf("HTTP %d", resp.StatusCode)}
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return ScrapeResponse{URL: url, Success: false, Error: err.Error()}
|
||||
}
|
||||
|
||||
html := string(body)
|
||||
title := extractHTMLTitle(html)
|
||||
content := extractTextContent(html)
|
||||
|
||||
if len(content) < 100 {
|
||||
return ScrapeResponse{URL: url, Success: false, Error: "Insufficient content"}
|
||||
}
|
||||
|
||||
return ScrapeResponse{
|
||||
URL: url,
|
||||
Title: title,
|
||||
Content: truncate(content, 15000),
|
||||
Success: true,
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
titleRegex = regexp.MustCompile(`<title[^>]*>([^<]+)</title>`)
|
||||
scriptRegex = regexp.MustCompile(`(?s)<script[^>]*>.*?</script>`)
|
||||
styleRegex = regexp.MustCompile(`(?s)<style[^>]*>.*?</style>`)
|
||||
tagRegex = regexp.MustCompile(`<[^>]+>`)
|
||||
spaceRegex = regexp.MustCompile(`\s+`)
|
||||
)
|
||||
|
||||
func extractHTMLTitle(html string) string {
|
||||
matches := titleRegex.FindStringSubmatch(html)
|
||||
if len(matches) > 1 {
|
||||
return strings.TrimSpace(matches[1])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractTextContent(html string) string {
|
||||
bodyStart := strings.Index(strings.ToLower(html), "<body")
|
||||
bodyEnd := strings.Index(strings.ToLower(html), "</body>")
|
||||
|
||||
if bodyStart != -1 && bodyEnd != -1 && bodyEnd > bodyStart {
|
||||
html = html[bodyStart:bodyEnd]
|
||||
}
|
||||
|
||||
html = scriptRegex.ReplaceAllString(html, "")
|
||||
html = styleRegex.ReplaceAllString(html, "")
|
||||
html = tagRegex.ReplaceAllString(html, " ")
|
||||
html = spaceRegex.ReplaceAllString(html, " ")
|
||||
|
||||
return strings.TrimSpace(html)
|
||||
}
|
||||
|
||||
func extractMarkdown(response string) string {
|
||||
if idx := strings.Index(response, `"raw_markdown"`); idx != -1 {
|
||||
start := strings.Index(response[idx:], `"`) + idx + 1
|
||||
start = strings.Index(response[start:], `"`) + start + 1
|
||||
end := strings.Index(response[start:], `"`)
|
||||
if end != -1 {
|
||||
return response[start : start+end]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractTitle(response string) string {
|
||||
if idx := strings.Index(response, `"title"`); idx != -1 {
|
||||
start := strings.Index(response[idx:], `"`) + idx + 1
|
||||
start = strings.Index(response[start:], `"`) + start + 1
|
||||
end := strings.Index(response[start:], `"`)
|
||||
if end != -1 {
|
||||
return response[start : start+end]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func truncate(s string, maxLen int) string {
|
||||
if len(s) <= maxLen {
|
||||
return s
|
||||
}
|
||||
return s[:maxLen]
|
||||
}
|
||||
|
||||
func init() {
|
||||
if os.Getenv("PORT") == "" {
|
||||
os.Setenv("PORT", "3021")
|
||||
}
|
||||
}
|
||||
143
backend/cmd/search-svc/main.go
Normal file
143
backend/cmd/search-svc/main.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/search"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
searchClient := search.NewSearXNGClient(cfg)
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/api/v1/search", func(c *fiber.Ctx) error {
|
||||
query := c.Query("q")
|
||||
if query == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Query parameter 'q' is required"})
|
||||
}
|
||||
|
||||
opts := &search.SearchOptions{}
|
||||
|
||||
if engines := c.Query("engines"); engines != "" {
|
||||
opts.Engines = strings.Split(engines, ",")
|
||||
}
|
||||
if categories := c.Query("categories"); categories != "" {
|
||||
opts.Categories = strings.Split(categories, ",")
|
||||
}
|
||||
if pageno := c.QueryInt("pageno", 1); pageno > 0 {
|
||||
opts.PageNo = pageno
|
||||
}
|
||||
if lang := c.Query("language"); lang != "" {
|
||||
opts.Language = lang
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout)
|
||||
defer cancel()
|
||||
|
||||
result, err := searchClient.Search(ctx, query, opts)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/search/images", func(c *fiber.Ctx) error {
|
||||
query := c.Query("q")
|
||||
if query == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Query parameter 'q' is required"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout)
|
||||
defer cancel()
|
||||
|
||||
result, err := searchClient.Search(ctx, query, &search.SearchOptions{
|
||||
Categories: []string{"images"},
|
||||
PageNo: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/search/videos", func(c *fiber.Ctx) error {
|
||||
query := c.Query("q")
|
||||
if query == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Query parameter 'q' is required"})
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout)
|
||||
defer cancel()
|
||||
|
||||
result, err := searchClient.Search(ctx, query, &search.SearchOptions{
|
||||
Categories: []string{"videos"},
|
||||
PageNo: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
app.Get("/api/v1/search/media", func(c *fiber.Ctx) error {
|
||||
query := c.Query("q")
|
||||
if query == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Query parameter 'q' is required"})
|
||||
}
|
||||
|
||||
maxImages := c.QueryInt("maxImages", 8)
|
||||
maxVideos := c.QueryInt("maxVideos", 6)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.SearchTimeout*2)
|
||||
defer cancel()
|
||||
|
||||
result, err := searchClient.SearchMedia(ctx, query, &search.MediaSearchOptions{
|
||||
MaxImages: maxImages,
|
||||
MaxVideos: maxVideos,
|
||||
})
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
||||
}
|
||||
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
port := cfg.SearchSvcPort
|
||||
log.Printf("search-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func init() {
|
||||
if os.Getenv("PORT") == "" {
|
||||
os.Setenv("PORT", "3001")
|
||||
}
|
||||
}
|
||||
624
backend/cmd/thread-svc/main.go
Normal file
624
backend/cmd/thread-svc/main.go
Normal file
@@ -0,0 +1,624 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/db"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/internal/pages"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/gooseek/backend/pkg/middleware"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
var database *db.PostgresDB
|
||||
var threadRepo *db.ThreadRepository
|
||||
var spaceRepo *db.SpaceRepository
|
||||
var memoryRepo *db.MemoryRepository
|
||||
var pageRepo *db.PageRepository
|
||||
|
||||
if cfg.DatabaseURL != "" {
|
||||
maxRetries := 30
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
database, err = db.NewPostgresDB(cfg.DatabaseURL)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
log.Printf("Waiting for database (attempt %d/%d): %v", i+1, maxRetries, err)
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal("Database required for thread-svc:", err)
|
||||
}
|
||||
log.Println("PostgreSQL connected successfully")
|
||||
defer database.Close()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
if err := database.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Base migrations warning: %v", err)
|
||||
}
|
||||
|
||||
spaceRepo = db.NewSpaceRepository(database)
|
||||
if err := spaceRepo.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Space migrations warning: %v", err)
|
||||
}
|
||||
|
||||
threadRepo = db.NewThreadRepository(database)
|
||||
if err := threadRepo.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Thread migrations warning: %v", err)
|
||||
}
|
||||
|
||||
memoryRepo = db.NewMemoryRepository(database)
|
||||
if err := memoryRepo.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Memory migrations warning: %v", err)
|
||||
}
|
||||
|
||||
pageRepo = db.NewPageRepository(database)
|
||||
if err := pageRepo.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Page migrations warning: %v", err)
|
||||
}
|
||||
|
||||
log.Println("PostgreSQL connected, all migrations complete")
|
||||
} else {
|
||||
log.Fatal("DATABASE_URL required for thread-svc")
|
||||
}
|
||||
|
||||
var llmClient llm.Client
|
||||
if cfg.OpenAIAPIKey != "" {
|
||||
llmClient, err = llm.NewClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
ModelKey: "gpt-4o-mini",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to create LLM client: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 10 * 1024 * 1024,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/ready", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ready"})
|
||||
})
|
||||
|
||||
threads := app.Group("/api/v1/threads", middleware.JWT(middleware.JWTConfig{
|
||||
Secret: cfg.JWTSecret,
|
||||
AuthSvcURL: cfg.AuthSvcURL,
|
||||
AllowGuest: true,
|
||||
}))
|
||||
|
||||
threads.Get("/", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.JSON(fiber.Map{"threads": []interface{}{}})
|
||||
}
|
||||
|
||||
limit := c.QueryInt("limit", 50)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
threadList, err := threadRepo.GetByUserID(c.Context(), userID, limit, offset)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get threads"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"threads": threadList})
|
||||
})
|
||||
|
||||
threads.Post("/", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Authentication required"})
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Title string `json:"title"`
|
||||
FocusMode string `json:"focusMode"`
|
||||
SpaceID *string `json:"spaceId"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
thread := &db.Thread{
|
||||
UserID: userID,
|
||||
SpaceID: req.SpaceID,
|
||||
Title: req.Title,
|
||||
FocusMode: req.FocusMode,
|
||||
}
|
||||
|
||||
if thread.Title == "" {
|
||||
thread.Title = "New Thread"
|
||||
}
|
||||
if thread.FocusMode == "" {
|
||||
thread.FocusMode = "all"
|
||||
}
|
||||
|
||||
if err := threadRepo.Create(c.Context(), thread); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to create thread"})
|
||||
}
|
||||
|
||||
return c.Status(201).JSON(thread)
|
||||
})
|
||||
|
||||
threads.Get("/:id", func(c *fiber.Ctx) error {
|
||||
threadID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
thread, err := threadRepo.GetByID(c.Context(), threadID)
|
||||
if err != nil || thread == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
|
||||
}
|
||||
|
||||
if thread.UserID != userID && !thread.IsPublic {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
messages, _ := threadRepo.GetMessages(c.Context(), threadID, 100, 0)
|
||||
thread.Messages = messages
|
||||
|
||||
return c.JSON(thread)
|
||||
})
|
||||
|
||||
threads.Post("/:id/messages", func(c *fiber.Ctx) error {
|
||||
threadID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
thread, err := threadRepo.GetByID(c.Context(), threadID)
|
||||
if err != nil || thread == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
|
||||
}
|
||||
|
||||
if thread.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
var req struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
Sources []db.ThreadSource `json:"sources"`
|
||||
Widgets []map[string]interface{} `json:"widgets"`
|
||||
RelatedQuestions []string `json:"relatedQuestions"`
|
||||
Model string `json:"model"`
|
||||
TokensUsed int `json:"tokensUsed"`
|
||||
}
|
||||
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
msg := &db.ThreadMessage{
|
||||
ThreadID: threadID,
|
||||
Role: req.Role,
|
||||
Content: req.Content,
|
||||
Sources: req.Sources,
|
||||
Widgets: req.Widgets,
|
||||
RelatedQuestions: req.RelatedQuestions,
|
||||
Model: req.Model,
|
||||
TokensUsed: req.TokensUsed,
|
||||
}
|
||||
|
||||
if err := threadRepo.AddMessage(c.Context(), msg); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to add message"})
|
||||
}
|
||||
|
||||
if thread.Title == "New Thread" && req.Role == "user" {
|
||||
threadRepo.GenerateTitle(c.Context(), threadID, req.Content)
|
||||
}
|
||||
|
||||
return c.Status(201).JSON(msg)
|
||||
})
|
||||
|
||||
threads.Post("/:id/share", func(c *fiber.Ctx) error {
|
||||
threadID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
thread, err := threadRepo.GetByID(c.Context(), threadID)
|
||||
if err != nil || thread == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
|
||||
}
|
||||
|
||||
if thread.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
shareID := generateShareID()
|
||||
if err := threadRepo.SetShareID(c.Context(), threadID, shareID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to share thread"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"shareId": shareID,
|
||||
"shareUrl": fmt.Sprintf("/share/%s", shareID),
|
||||
})
|
||||
})
|
||||
|
||||
threads.Delete("/:id", func(c *fiber.Ctx) error {
|
||||
threadID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
thread, err := threadRepo.GetByID(c.Context(), threadID)
|
||||
if err != nil || thread == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
|
||||
}
|
||||
|
||||
if thread.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
if err := threadRepo.Delete(c.Context(), threadID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to delete thread"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
share := app.Group("/api/v1/share")
|
||||
|
||||
share.Get("/:shareId", func(c *fiber.Ctx) error {
|
||||
shareID := c.Params("shareId")
|
||||
|
||||
thread, err := threadRepo.GetByShareID(c.Context(), shareID)
|
||||
if err != nil || thread == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Shared thread not found"})
|
||||
}
|
||||
|
||||
messages, _ := threadRepo.GetMessages(c.Context(), thread.ID, 100, 0)
|
||||
thread.Messages = messages
|
||||
|
||||
return c.JSON(thread)
|
||||
})
|
||||
|
||||
spaces := app.Group("/api/v1/spaces", middleware.JWT(middleware.JWTConfig{
|
||||
Secret: cfg.JWTSecret,
|
||||
AuthSvcURL: cfg.AuthSvcURL,
|
||||
AllowGuest: false,
|
||||
}))
|
||||
|
||||
spaces.Get("/", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
spaceList, err := spaceRepo.GetByUserID(c.Context(), userID)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get spaces"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"spaces": spaceList})
|
||||
})
|
||||
|
||||
spaces.Post("/", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
var req db.Space
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
req.UserID = userID
|
||||
if req.Name == "" {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Name required"})
|
||||
}
|
||||
|
||||
if err := spaceRepo.Create(c.Context(), &req); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to create space"})
|
||||
}
|
||||
|
||||
return c.Status(201).JSON(req)
|
||||
})
|
||||
|
||||
spaces.Get("/:id", func(c *fiber.Ctx) error {
|
||||
spaceID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
space, err := spaceRepo.GetByID(c.Context(), spaceID)
|
||||
if err != nil || space == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Space not found"})
|
||||
}
|
||||
|
||||
if space.UserID != userID && !space.IsPublic {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
return c.JSON(space)
|
||||
})
|
||||
|
||||
spaces.Put("/:id", func(c *fiber.Ctx) error {
|
||||
spaceID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
space, err := spaceRepo.GetByID(c.Context(), spaceID)
|
||||
if err != nil || space == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Space not found"})
|
||||
}
|
||||
|
||||
if space.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
var req db.Space
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
req.ID = spaceID
|
||||
req.UserID = userID
|
||||
|
||||
if err := spaceRepo.Update(c.Context(), &req); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to update space"})
|
||||
}
|
||||
|
||||
return c.JSON(req)
|
||||
})
|
||||
|
||||
spaces.Delete("/:id", func(c *fiber.Ctx) error {
|
||||
spaceID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
space, err := spaceRepo.GetByID(c.Context(), spaceID)
|
||||
if err != nil || space == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Space not found"})
|
||||
}
|
||||
|
||||
if space.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
if err := spaceRepo.Delete(c.Context(), spaceID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to delete space"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
memory := app.Group("/api/v1/memory", middleware.JWT(middleware.JWTConfig{
|
||||
Secret: cfg.JWTSecret,
|
||||
AuthSvcURL: cfg.AuthSvcURL,
|
||||
AllowGuest: false,
|
||||
}))
|
||||
|
||||
memory.Get("/", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
memType := c.Query("type", "")
|
||||
|
||||
memories, err := memoryRepo.GetByUserID(c.Context(), userID, memType, 50)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get memories"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"memories": memories})
|
||||
})
|
||||
|
||||
memory.Post("/", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
var req db.UserMemory
|
||||
if err := c.BodyParser(&req); err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
||||
}
|
||||
|
||||
req.UserID = userID
|
||||
|
||||
if err := memoryRepo.Save(c.Context(), &req); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to save memory"})
|
||||
}
|
||||
|
||||
return c.Status(201).JSON(req)
|
||||
})
|
||||
|
||||
memory.Get("/context", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
ctx, err := memoryRepo.GetContextForUser(c.Context(), userID)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get context"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"context": ctx})
|
||||
})
|
||||
|
||||
memory.Delete("/:id", func(c *fiber.Ctx) error {
|
||||
memID := c.Params("id")
|
||||
|
||||
if err := memoryRepo.Delete(c.Context(), memID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to delete memory"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
pagesAPI := app.Group("/api/v1/pages", middleware.JWT(middleware.JWTConfig{
|
||||
Secret: cfg.JWTSecret,
|
||||
AuthSvcURL: cfg.AuthSvcURL,
|
||||
AllowGuest: true,
|
||||
}))
|
||||
|
||||
pagesAPI.Get("/", func(c *fiber.Ctx) error {
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.JSON(fiber.Map{"pages": []interface{}{}})
|
||||
}
|
||||
|
||||
limit := c.QueryInt("limit", 50)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
pageList, err := pageRepo.GetByUserID(c.Context(), userID, limit, offset)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get pages"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"pages": pageList})
|
||||
})
|
||||
|
||||
pagesAPI.Post("/from-thread/:threadId", func(c *fiber.Ctx) error {
|
||||
threadID := c.Params("threadId")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Authentication required"})
|
||||
}
|
||||
|
||||
thread, err := threadRepo.GetByID(c.Context(), threadID)
|
||||
if err != nil || thread == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Thread not found"})
|
||||
}
|
||||
|
||||
if thread.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
messages, _ := threadRepo.GetMessages(c.Context(), threadID, 100, 0)
|
||||
|
||||
var query, answer string
|
||||
for _, msg := range messages {
|
||||
if msg.Role == "user" && query == "" {
|
||||
query = msg.Content
|
||||
}
|
||||
if msg.Role == "assistant" {
|
||||
answer += msg.Content + "\n\n"
|
||||
}
|
||||
}
|
||||
|
||||
if llmClient == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "LLM not configured"})
|
||||
}
|
||||
|
||||
generator := pages.NewPageGenerator(pages.PageGeneratorConfig{
|
||||
LLMClient: llmClient,
|
||||
Locale: c.Query("locale", "en"),
|
||||
})
|
||||
|
||||
page, err := generator.GenerateFromThread(c.Context(), query, answer, nil)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to generate page"})
|
||||
}
|
||||
|
||||
page.UserID = userID
|
||||
page.ThreadID = threadID
|
||||
|
||||
if err := pageRepo.Create(c.Context(), page); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to save page"})
|
||||
}
|
||||
|
||||
return c.Status(201).JSON(page)
|
||||
})
|
||||
|
||||
pagesAPI.Get("/:id", func(c *fiber.Ctx) error {
|
||||
pageID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
page, err := pageRepo.GetByID(c.Context(), pageID)
|
||||
if err != nil || page == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
|
||||
}
|
||||
|
||||
if page.UserID != userID && !page.IsPublic {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
return c.JSON(page)
|
||||
})
|
||||
|
||||
pagesAPI.Post("/:id/share", func(c *fiber.Ctx) error {
|
||||
pageID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
page, err := pageRepo.GetByID(c.Context(), pageID)
|
||||
if err != nil || page == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
|
||||
}
|
||||
|
||||
if page.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
shareID := generateShareID()
|
||||
if err := pageRepo.SetShareID(c.Context(), pageID, shareID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to share page"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"shareId": shareID,
|
||||
"shareUrl": fmt.Sprintf("/page/%s", shareID),
|
||||
})
|
||||
})
|
||||
|
||||
pagesAPI.Get("/share/:shareId", func(c *fiber.Ctx) error {
|
||||
shareID := c.Params("shareId")
|
||||
|
||||
page, err := pageRepo.GetByShareID(c.Context(), shareID)
|
||||
if err != nil || page == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
|
||||
}
|
||||
|
||||
pageRepo.IncrementViewCount(c.Context(), page.ID)
|
||||
|
||||
return c.JSON(page)
|
||||
})
|
||||
|
||||
pagesAPI.Delete("/:id", func(c *fiber.Ctx) error {
|
||||
pageID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
page, err := pageRepo.GetByID(c.Context(), pageID)
|
||||
if err != nil || page == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "Page not found"})
|
||||
}
|
||||
|
||||
if page.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
if err := pageRepo.Delete(c.Context(), pageID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to delete page"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
port := getEnvInt("THREAD_SVC_PORT", 3027)
|
||||
log.Printf("thread-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func generateShareID() string {
|
||||
b := make([]byte, 8)
|
||||
rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
Reference in New Issue
Block a user