feat: Go backend, enhanced search, new widgets, Docker deploy
Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
This commit is contained in:
home
307
backend/cmd/file-svc/main.go
Normal file
307
backend/cmd/file-svc/main.go
Normal file
@@ -0,0 +1,307 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
"github.com/gofiber/fiber/v2/middleware/cors"
|
||||
"github.com/gofiber/fiber/v2/middleware/logger"
|
||||
"github.com/gooseek/backend/internal/db"
|
||||
"github.com/gooseek/backend/internal/files"
|
||||
"github.com/gooseek/backend/internal/llm"
|
||||
"github.com/gooseek/backend/pkg/config"
|
||||
"github.com/gooseek/backend/pkg/middleware"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg, err := config.Load()
|
||||
if err != nil {
|
||||
log.Fatal("Failed to load config:", err)
|
||||
}
|
||||
|
||||
var database *db.PostgresDB
|
||||
var fileRepo *db.FileRepository
|
||||
|
||||
if cfg.DatabaseURL != "" {
|
||||
database, err = db.NewPostgresDB(cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Printf("Database unavailable: %v", err)
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if err := database.RunMigrations(ctx); err != nil {
|
||||
log.Printf("Migration warning: %v", err)
|
||||
}
|
||||
cancel()
|
||||
defer database.Close()
|
||||
|
||||
fileRepo = db.NewFileRepository(database)
|
||||
log.Println("PostgreSQL connected")
|
||||
}
|
||||
}
|
||||
|
||||
var llmClient llm.Client
|
||||
if cfg.OpenAIAPIKey != "" {
|
||||
llmClient, err = llm.NewClient(llm.ProviderConfig{
|
||||
ProviderID: "openai",
|
||||
ModelKey: "gpt-4o",
|
||||
APIKey: cfg.OpenAIAPIKey,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to create OpenAI client: %v", err)
|
||||
}
|
||||
} else if cfg.AnthropicAPIKey != "" {
|
||||
llmClient, err = llm.NewClient(llm.ProviderConfig{
|
||||
ProviderID: "anthropic",
|
||||
ModelKey: "claude-3-5-sonnet-20241022",
|
||||
APIKey: cfg.AnthropicAPIKey,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to create Anthropic client: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
storagePath := os.Getenv("FILE_STORAGE_PATH")
|
||||
if storagePath == "" {
|
||||
storagePath = "/tmp/gooseek-files"
|
||||
}
|
||||
|
||||
var fileAnalyzer *files.FileAnalyzer
|
||||
if llmClient != nil {
|
||||
fileAnalyzer = files.NewFileAnalyzer(llmClient, storagePath)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{
|
||||
BodyLimit: 100 * 1024 * 1024,
|
||||
ReadTimeout: 60 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
})
|
||||
|
||||
app.Use(logger.New())
|
||||
app.Use(cors.New())
|
||||
|
||||
if cfg.JWTSecret != "" || cfg.AuthSvcURL != "" {
|
||||
app.Use(middleware.JWT(middleware.JWTConfig{
|
||||
Secret: cfg.JWTSecret,
|
||||
AuthSvcURL: cfg.AuthSvcURL,
|
||||
AllowGuest: false,
|
||||
SkipPaths: []string{"/health", "/ready"},
|
||||
}))
|
||||
}
|
||||
|
||||
app.Get("/health", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ok"})
|
||||
})
|
||||
|
||||
app.Get("/ready", func(c *fiber.Ctx) error {
|
||||
return c.JSON(fiber.Map{"status": "ready"})
|
||||
})
|
||||
|
||||
api := app.Group("/api/v1/files")
|
||||
|
||||
api.Post("/upload", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil || fileAnalyzer == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Unauthorized"})
|
||||
}
|
||||
|
||||
file, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "No file uploaded"})
|
||||
}
|
||||
|
||||
if file.Size > 50*1024*1024 {
|
||||
return c.Status(400).JSON(fiber.Map{"error": "File too large (max 50MB)"})
|
||||
}
|
||||
|
||||
f, err := file.Open()
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to read file"})
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
storagePath, fileSize, err := fileAnalyzer.SaveFile(file.Filename, f)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to save file"})
|
||||
}
|
||||
|
||||
buf := make([]byte, 512)
|
||||
f.Seek(0, 0)
|
||||
f.Read(buf)
|
||||
mimeType := files.DetectMimeType(file.Filename, buf)
|
||||
|
||||
uploadedFile := &db.UploadedFile{
|
||||
UserID: userID,
|
||||
Filename: file.Filename,
|
||||
FileType: mimeType,
|
||||
FileSize: fileSize,
|
||||
StoragePath: storagePath,
|
||||
Metadata: map[string]interface{}{},
|
||||
}
|
||||
|
||||
if err := fileRepo.Create(c.Context(), uploadedFile); err != nil {
|
||||
fileAnalyzer.DeleteFile(storagePath)
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to save file record"})
|
||||
}
|
||||
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
result, err := fileAnalyzer.AnalyzeFile(ctx, storagePath, mimeType)
|
||||
if err != nil {
|
||||
log.Printf("File analysis failed for %s: %v", uploadedFile.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
fileRepo.UpdateExtractedText(ctx, uploadedFile.ID, result.ExtractedText)
|
||||
}()
|
||||
|
||||
return c.Status(201).JSON(fiber.Map{
|
||||
"id": uploadedFile.ID,
|
||||
"filename": uploadedFile.Filename,
|
||||
"fileType": uploadedFile.FileType,
|
||||
"fileSize": uploadedFile.FileSize,
|
||||
"status": "processing",
|
||||
})
|
||||
})
|
||||
|
||||
api.Get("/", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
userID := middleware.GetUserID(c)
|
||||
if userID == "" {
|
||||
return c.Status(401).JSON(fiber.Map{"error": "Unauthorized"})
|
||||
}
|
||||
|
||||
limit := c.QueryInt("limit", 50)
|
||||
offset := c.QueryInt("offset", 0)
|
||||
|
||||
files, err := fileRepo.GetByUserID(c.Context(), userID, limit, offset)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to get files"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{"files": files})
|
||||
})
|
||||
|
||||
api.Get("/:id", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
return c.JSON(file)
|
||||
})
|
||||
|
||||
api.Get("/:id/content", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
return c.JSON(fiber.Map{
|
||||
"id": file.ID,
|
||||
"filename": file.Filename,
|
||||
"extractedText": file.ExtractedText,
|
||||
})
|
||||
})
|
||||
|
||||
api.Post("/:id/analyze", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil || fileAnalyzer == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
result, err := fileAnalyzer.AnalyzeFile(c.Context(), file.StoragePath, file.FileType)
|
||||
if err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Analysis failed: " + err.Error()})
|
||||
}
|
||||
|
||||
fileRepo.UpdateExtractedText(c.Context(), fileID, result.ExtractedText)
|
||||
|
||||
return c.JSON(result)
|
||||
})
|
||||
|
||||
api.Delete("/:id", func(c *fiber.Ctx) error {
|
||||
if fileRepo == nil || fileAnalyzer == nil {
|
||||
return c.Status(503).JSON(fiber.Map{"error": "Service unavailable"})
|
||||
}
|
||||
|
||||
fileID := c.Params("id")
|
||||
userID := middleware.GetUserID(c)
|
||||
|
||||
file, err := fileRepo.GetByID(c.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
return c.Status(404).JSON(fiber.Map{"error": "File not found"})
|
||||
}
|
||||
|
||||
if file.UserID != userID {
|
||||
return c.Status(403).JSON(fiber.Map{"error": "Access denied"})
|
||||
}
|
||||
|
||||
fileAnalyzer.DeleteFile(file.StoragePath)
|
||||
|
||||
if err := fileRepo.Delete(c.Context(), fileID); err != nil {
|
||||
return c.Status(500).JSON(fiber.Map{"error": "Failed to delete file"})
|
||||
}
|
||||
|
||||
return c.Status(204).Send(nil)
|
||||
})
|
||||
|
||||
port := getEnvInt("FILE_SVC_PORT", 3026)
|
||||
log.Printf("file-svc listening on :%d", port)
|
||||
log.Fatal(app.Listen(fmt.Sprintf(":%d", port)))
|
||||
}
|
||||
|
||||
func getEnvInt(key string, defaultValue int) int {
|
||||
if val := os.Getenv(key); val != "" {
|
||||
var result int
|
||||
if _, err := fmt.Sscanf(val, "%d", &result); err == nil {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
Reference in New Issue
Block a user