06fe57c765
Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
556 lines
14 KiB
Go
556 lines
14 KiB
Go
package browser
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/gofiber/fiber/v2"
|
|
"github.com/gofiber/fiber/v2/middleware/cors"
|
|
"github.com/gofiber/fiber/v2/middleware/logger"
|
|
)
|
|
|
|
type BrowserServer struct {
|
|
browser *PlaywrightBrowser
|
|
sessions map[string]*ManagedSession
|
|
mu sync.RWMutex
|
|
config ServerConfig
|
|
}
|
|
|
|
type ServerConfig struct {
|
|
Port int
|
|
MaxSessions int
|
|
SessionTimeout time.Duration
|
|
CleanupInterval time.Duration
|
|
}
|
|
|
|
type ManagedSession struct {
|
|
*BrowserSession
|
|
LastActive time.Time
|
|
Actions []ActionLog
|
|
}
|
|
|
|
type ActionLog struct {
|
|
Action string `json:"action"`
|
|
Params string `json:"params"`
|
|
Success bool `json:"success"`
|
|
Error string `json:"error,omitempty"`
|
|
Duration int64 `json:"durationMs"`
|
|
Timestamp time.Time `json:"timestamp"`
|
|
}
|
|
|
|
type BrowserRequest struct {
|
|
Method string `json:"method"`
|
|
Params map[string]interface{} `json:"params"`
|
|
}
|
|
|
|
func NewBrowserServer(cfg ServerConfig) *BrowserServer {
|
|
if cfg.Port == 0 {
|
|
cfg.Port = 3050
|
|
}
|
|
if cfg.MaxSessions == 0 {
|
|
cfg.MaxSessions = 20
|
|
}
|
|
if cfg.SessionTimeout == 0 {
|
|
cfg.SessionTimeout = 30 * time.Minute
|
|
}
|
|
if cfg.CleanupInterval == 0 {
|
|
cfg.CleanupInterval = 5 * time.Minute
|
|
}
|
|
|
|
return &BrowserServer{
|
|
browser: NewPlaywrightBrowser(Config{
|
|
DefaultTimeout: 30 * time.Second,
|
|
Headless: true,
|
|
}),
|
|
sessions: make(map[string]*ManagedSession),
|
|
config: cfg,
|
|
}
|
|
}
|
|
|
|
func (s *BrowserServer) Start(ctx context.Context) error {
|
|
go s.cleanupLoop(ctx)
|
|
|
|
app := fiber.New(fiber.Config{
|
|
BodyLimit: 50 * 1024 * 1024,
|
|
ReadTimeout: 2 * time.Minute,
|
|
WriteTimeout: 2 * time.Minute,
|
|
})
|
|
|
|
app.Use(logger.New())
|
|
app.Use(cors.New())
|
|
|
|
app.Get("/health", func(c *fiber.Ctx) error {
|
|
return c.JSON(fiber.Map{"status": "ok", "sessions": len(s.sessions)})
|
|
})
|
|
|
|
app.Post("/api/browser", s.handleBrowserCommand)
|
|
|
|
app.Post("/api/session/new", s.handleNewSession)
|
|
app.Delete("/api/session/:id", s.handleCloseSession)
|
|
app.Get("/api/session/:id", s.handleGetSession)
|
|
app.Get("/api/sessions", s.handleListSessions)
|
|
|
|
app.Post("/api/action", s.handleAction)
|
|
|
|
log.Printf("[BrowserServer] Starting on port %d", s.config.Port)
|
|
return app.Listen(fmt.Sprintf(":%d", s.config.Port))
|
|
}
|
|
|
|
func (s *BrowserServer) handleBrowserCommand(c *fiber.Ctx) error {
|
|
var req BrowserRequest
|
|
if err := c.BodyParser(&req); err != nil {
|
|
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
|
|
sessionID, _ := req.Params["sessionId"].(string)
|
|
|
|
s.mu.Lock()
|
|
if session, ok := s.sessions[sessionID]; ok {
|
|
session.LastActive = time.Now()
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
start := time.Now()
|
|
result, err := s.executeMethod(ctx, req.Method, req.Params)
|
|
|
|
s.mu.Lock()
|
|
if session, ok := s.sessions[sessionID]; ok {
|
|
paramsJSON, _ := json.Marshal(req.Params)
|
|
session.Actions = append(session.Actions, ActionLog{
|
|
Action: req.Method,
|
|
Params: string(paramsJSON),
|
|
Success: err == nil,
|
|
Error: errToString(err),
|
|
Duration: time.Since(start).Milliseconds(),
|
|
Timestamp: time.Now(),
|
|
})
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
if err != nil {
|
|
return c.JSON(fiber.Map{
|
|
"success": false,
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
|
|
return c.JSON(result)
|
|
}
|
|
|
|
func (s *BrowserServer) executeMethod(ctx context.Context, method string, params map[string]interface{}) (map[string]interface{}, error) {
|
|
sessionID, _ := params["sessionId"].(string)
|
|
|
|
switch method {
|
|
case "browser.newContext":
|
|
opts := SessionOptions{
|
|
Headless: getBool(params, "headless"),
|
|
}
|
|
if viewport, ok := params["viewport"].(map[string]interface{}); ok {
|
|
opts.Viewport = &Viewport{
|
|
Width: getInt(viewport, "width"),
|
|
Height: getInt(viewport, "height"),
|
|
}
|
|
}
|
|
if ua, ok := params["userAgent"].(string); ok {
|
|
opts.UserAgent = ua
|
|
}
|
|
if proxy, ok := params["proxy"].(string); ok {
|
|
opts.ProxyURL = proxy
|
|
}
|
|
if rv, ok := params["recordVideo"].(map[string]interface{}); ok {
|
|
_ = rv
|
|
opts.RecordVideo = true
|
|
}
|
|
|
|
session, err := s.browser.NewSession(ctx, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
s.mu.Lock()
|
|
s.sessions[session.ID] = &ManagedSession{
|
|
BrowserSession: session,
|
|
LastActive: time.Now(),
|
|
Actions: make([]ActionLog, 0),
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
return map[string]interface{}{
|
|
"sessionId": session.ID,
|
|
"contextId": session.ContextID,
|
|
"pageId": session.PageID,
|
|
}, nil
|
|
|
|
case "browser.closeContext":
|
|
err := s.browser.CloseSession(ctx, sessionID)
|
|
s.mu.Lock()
|
|
delete(s.sessions, sessionID)
|
|
s.mu.Unlock()
|
|
return map[string]interface{}{"success": err == nil}, err
|
|
|
|
case "page.goto":
|
|
url, _ := params["url"].(string)
|
|
opts := NavigateOptions{
|
|
Timeout: getInt(params, "timeout"),
|
|
WaitUntil: getString(params, "waitUntil"),
|
|
}
|
|
result, err := s.browser.Navigate(ctx, sessionID, url, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]interface{}{
|
|
"success": result.Success,
|
|
"url": result.PageURL,
|
|
"title": result.PageTitle,
|
|
}, nil
|
|
|
|
case "page.click":
|
|
selector, _ := params["selector"].(string)
|
|
opts := ClickOptions{
|
|
Button: getString(params, "button"),
|
|
ClickCount: getInt(params, "clickCount"),
|
|
Timeout: getInt(params, "timeout"),
|
|
Force: getBool(params, "force"),
|
|
}
|
|
result, err := s.browser.Click(ctx, sessionID, selector, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]interface{}{
|
|
"success": result.Success,
|
|
"screenshot": result.Screenshot,
|
|
}, nil
|
|
|
|
case "page.type":
|
|
selector, _ := params["selector"].(string)
|
|
text, _ := params["text"].(string)
|
|
opts := TypeOptions{
|
|
Delay: getInt(params, "delay"),
|
|
Timeout: getInt(params, "timeout"),
|
|
}
|
|
_, err := s.browser.Type(ctx, sessionID, selector, text, opts)
|
|
return map[string]interface{}{"success": err == nil}, err
|
|
|
|
case "page.fill":
|
|
selector, _ := params["selector"].(string)
|
|
value, _ := params["value"].(string)
|
|
_, err := s.browser.Fill(ctx, sessionID, selector, value)
|
|
return map[string]interface{}{"success": err == nil}, err
|
|
|
|
case "page.screenshot":
|
|
opts := ScreenshotOptions{
|
|
FullPage: getBool(params, "fullPage"),
|
|
Selector: getString(params, "selector"),
|
|
Format: getString(params, "type"),
|
|
Quality: getInt(params, "quality"),
|
|
}
|
|
result, err := s.browser.Screenshot(ctx, sessionID, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]interface{}{
|
|
"data": result.Data,
|
|
"path": result.Path,
|
|
}, nil
|
|
|
|
case "page.textContent":
|
|
selector, _ := params["selector"].(string)
|
|
text, err := s.browser.ExtractText(ctx, sessionID, selector)
|
|
return map[string]interface{}{"text": text}, err
|
|
|
|
case "page.innerHTML":
|
|
selector, _ := params["selector"].(string)
|
|
html, err := s.browser.ExtractHTML(ctx, sessionID, selector)
|
|
return map[string]interface{}{"html": html}, err
|
|
|
|
case "page.waitForSelector":
|
|
selector, _ := params["selector"].(string)
|
|
opts := WaitOptions{
|
|
Timeout: getInt(params, "timeout"),
|
|
State: getString(params, "state"),
|
|
}
|
|
err := s.browser.WaitForSelector(ctx, sessionID, selector, opts)
|
|
return map[string]interface{}{"success": err == nil}, err
|
|
|
|
case "page.waitForNavigation":
|
|
opts := WaitOptions{
|
|
Timeout: getInt(params, "timeout"),
|
|
WaitUntil: getString(params, "waitUntil"),
|
|
}
|
|
err := s.browser.WaitForNavigation(ctx, sessionID, opts)
|
|
return map[string]interface{}{"success": err == nil}, err
|
|
|
|
case "page.evaluate":
|
|
expression, _ := params["expression"].(string)
|
|
result, err := s.browser.Evaluate(ctx, sessionID, expression)
|
|
return map[string]interface{}{"result": result}, err
|
|
|
|
case "page.selectOption":
|
|
selector, _ := params["selector"].(string)
|
|
values := getStringArray(params, "values")
|
|
_, err := s.browser.Select(ctx, sessionID, selector, values)
|
|
return map[string]interface{}{"success": err == nil}, err
|
|
|
|
case "page.info":
|
|
info, err := s.browser.GetPageInfo(ctx, sessionID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]interface{}{
|
|
"url": info.URL,
|
|
"title": info.Title,
|
|
"content": info.Content,
|
|
}, nil
|
|
|
|
case "page.pdf":
|
|
opts := PDFOptions{
|
|
Format: getString(params, "format"),
|
|
Landscape: getBool(params, "landscape"),
|
|
PrintBackground: getBool(params, "printBackground"),
|
|
}
|
|
data, err := s.browser.PDF(ctx, sessionID, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return map[string]interface{}{
|
|
"data": data,
|
|
}, nil
|
|
|
|
default:
|
|
return nil, fmt.Errorf("unknown method: %s", method)
|
|
}
|
|
}
|
|
|
|
func (s *BrowserServer) handleNewSession(c *fiber.Ctx) error {
|
|
var req struct {
|
|
Headless bool `json:"headless"`
|
|
Viewport *Viewport `json:"viewport,omitempty"`
|
|
UserAgent string `json:"userAgent,omitempty"`
|
|
ProxyURL string `json:"proxyUrl,omitempty"`
|
|
}
|
|
|
|
if err := c.BodyParser(&req); err != nil {
|
|
req.Headless = true
|
|
}
|
|
|
|
s.mu.RLock()
|
|
if len(s.sessions) >= s.config.MaxSessions {
|
|
s.mu.RUnlock()
|
|
return c.Status(http.StatusTooManyRequests).JSON(fiber.Map{
|
|
"error": "Maximum sessions limit reached",
|
|
})
|
|
}
|
|
s.mu.RUnlock()
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
|
|
session, err := s.browser.NewSession(ctx, SessionOptions{
|
|
Headless: req.Headless,
|
|
Viewport: req.Viewport,
|
|
UserAgent: req.UserAgent,
|
|
ProxyURL: req.ProxyURL,
|
|
})
|
|
if err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
s.mu.Lock()
|
|
s.sessions[session.ID] = &ManagedSession{
|
|
BrowserSession: session,
|
|
LastActive: time.Now(),
|
|
Actions: make([]ActionLog, 0),
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
return c.JSON(fiber.Map{
|
|
"sessionId": session.ID,
|
|
"contextId": session.ContextID,
|
|
"pageId": session.PageID,
|
|
})
|
|
}
|
|
|
|
func (s *BrowserServer) handleCloseSession(c *fiber.Ctx) error {
|
|
sessionID := c.Params("id")
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
err := s.browser.CloseSession(ctx, sessionID)
|
|
if err != nil {
|
|
return c.Status(404).JSON(fiber.Map{"error": err.Error()})
|
|
}
|
|
|
|
s.mu.Lock()
|
|
delete(s.sessions, sessionID)
|
|
s.mu.Unlock()
|
|
|
|
return c.JSON(fiber.Map{"success": true})
|
|
}
|
|
|
|
func (s *BrowserServer) handleGetSession(c *fiber.Ctx) error {
|
|
sessionID := c.Params("id")
|
|
|
|
s.mu.RLock()
|
|
session, ok := s.sessions[sessionID]
|
|
s.mu.RUnlock()
|
|
|
|
if !ok {
|
|
return c.Status(404).JSON(fiber.Map{"error": "Session not found"})
|
|
}
|
|
|
|
return c.JSON(fiber.Map{
|
|
"sessionId": session.ID,
|
|
"createdAt": session.CreatedAt,
|
|
"lastActive": session.LastActive,
|
|
"screenshots": session.Screenshots,
|
|
"actions": len(session.Actions),
|
|
})
|
|
}
|
|
|
|
func (s *BrowserServer) handleListSessions(c *fiber.Ctx) error {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
|
|
sessions := make([]map[string]interface{}, 0, len(s.sessions))
|
|
for _, session := range s.sessions {
|
|
sessions = append(sessions, map[string]interface{}{
|
|
"sessionId": session.ID,
|
|
"createdAt": session.CreatedAt,
|
|
"lastActive": session.LastActive,
|
|
"actions": len(session.Actions),
|
|
})
|
|
}
|
|
|
|
return c.JSON(fiber.Map{"sessions": sessions, "count": len(sessions)})
|
|
}
|
|
|
|
func (s *BrowserServer) handleAction(c *fiber.Ctx) error {
|
|
var req struct {
|
|
SessionID string `json:"sessionId"`
|
|
Action string `json:"action"`
|
|
Selector string `json:"selector,omitempty"`
|
|
URL string `json:"url,omitempty"`
|
|
Value string `json:"value,omitempty"`
|
|
Screenshot bool `json:"screenshot"`
|
|
}
|
|
|
|
if err := c.BodyParser(&req); err != nil {
|
|
return c.Status(400).JSON(fiber.Map{"error": "Invalid request"})
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
defer cancel()
|
|
|
|
s.mu.Lock()
|
|
if session, ok := s.sessions[req.SessionID]; ok {
|
|
session.LastActive = time.Now()
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
var result *ActionResponse
|
|
var err error
|
|
|
|
switch req.Action {
|
|
case "navigate":
|
|
result, err = s.browser.Navigate(ctx, req.SessionID, req.URL, NavigateOptions{Screenshot: req.Screenshot})
|
|
case "click":
|
|
result, err = s.browser.Click(ctx, req.SessionID, req.Selector, ClickOptions{Screenshot: req.Screenshot})
|
|
case "type":
|
|
result, err = s.browser.Type(ctx, req.SessionID, req.Selector, req.Value, TypeOptions{})
|
|
case "fill":
|
|
result, err = s.browser.Fill(ctx, req.SessionID, req.Selector, req.Value)
|
|
case "screenshot":
|
|
var screenshot *ScreenshotResult
|
|
screenshot, err = s.browser.Screenshot(ctx, req.SessionID, ScreenshotOptions{})
|
|
if err == nil {
|
|
result = &ActionResponse{Success: true, Screenshot: screenshot.Data}
|
|
}
|
|
case "extract":
|
|
var text string
|
|
text, err = s.browser.ExtractText(ctx, req.SessionID, req.Selector)
|
|
result = &ActionResponse{Success: err == nil, Data: text}
|
|
default:
|
|
return c.Status(400).JSON(fiber.Map{"error": "Unknown action: " + req.Action})
|
|
}
|
|
|
|
if err != nil {
|
|
return c.Status(500).JSON(fiber.Map{"error": err.Error(), "success": false})
|
|
}
|
|
|
|
return c.JSON(result)
|
|
}
|
|
|
|
func (s *BrowserServer) cleanupLoop(ctx context.Context) {
|
|
ticker := time.NewTicker(s.config.CleanupInterval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
s.cleanupExpiredSessions()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *BrowserServer) cleanupExpiredSessions() {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
now := time.Now()
|
|
for sessionID, session := range s.sessions {
|
|
if now.Sub(session.LastActive) > s.config.SessionTimeout {
|
|
log.Printf("[BrowserServer] Cleaning up expired session: %s", sessionID)
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
s.browser.CloseSession(ctx, sessionID)
|
|
cancel()
|
|
delete(s.sessions, sessionID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func errToString(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
return err.Error()
|
|
}
|
|
|
|
func getBool(m map[string]interface{}, key string) bool {
|
|
if v, ok := m[key].(bool); ok {
|
|
return v
|
|
}
|
|
return false
|
|
}
|
|
|
|
func getInt(m map[string]interface{}, key string) int {
|
|
if v, ok := m[key].(float64); ok {
|
|
return int(v)
|
|
}
|
|
if v, ok := m[key].(int); ok {
|
|
return v
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func getStringArray(m map[string]interface{}, key string) []string {
|
|
if v, ok := m[key].([]interface{}); ok {
|
|
result := make([]string, len(v))
|
|
for i, item := range v {
|
|
result[i], _ = item.(string)
|
|
}
|
|
return result
|
|
}
|
|
return nil
|
|
}
|