feat: Go backend, enhanced search, new widgets, Docker deploy

Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
2026-02-27 04:15:32 +03:00
parent 328d968f3f
commit 06fe57c765
285 changed files with 53132 additions and 1871 deletions
--- a/backend/deploy/docker/Dockerfile.agent-svc
+++ b/backend/deploy/docker/Dockerfile.agent-svc
@@ -0,0 +1,31 @@
+# Dockerfile for agent-svc only
+FROM golang:1.22-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/agent-svc ./cmd/agent-svc
+
+# Runtime stage
+FROM alpine:3.19
+
+RUN apk add --no-cache ca-certificates tzdata
+
+WORKDIR /app
+
+COPY --from=builder /bin/agent-svc /app/agent-svc
+
+ENV PORT=3018
+
+EXPOSE 3018
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:3018/health || exit 1
+
+CMD ["/app/agent-svc"]
--- a/backend/deploy/docker/Dockerfile.all
+++ b/backend/deploy/docker/Dockerfile.all
@@ -0,0 +1,50 @@
+# Multi-service Dockerfile - builds all services
+FROM golang:1.22-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+# Build all services
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/api-gateway ./cmd/api-gateway
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/agent-svc ./cmd/agent-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/chat-svc ./cmd/chat-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/search-svc ./cmd/search-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/llm-svc ./cmd/llm-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/scraper-svc ./cmd/scraper-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/discover-svc ./cmd/discover-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/collection-svc ./cmd/collection-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/file-svc ./cmd/file-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/thread-svc ./cmd/thread-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/computer-svc ./cmd/computer-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/finance-heatmap-svc ./cmd/finance-heatmap-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/learning-svc ./cmd/learning-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/labs-svc ./cmd/labs-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/podcast-svc ./cmd/podcast-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/browser-svc ./cmd/browser-svc
+
+# Runtime stage
+FROM alpine:3.19
+
+RUN apk add --no-cache ca-certificates tzdata
+
+WORKDIR /app
+
+COPY --from=builder /bin/* /app/
+
+# Default entrypoint - can be overridden
+ENV SERVICE=api-gateway
+ENV PORT=3015
+
+EXPOSE 3015 3018 3005 3001 3020 3021 3002 3025 3026 3027 3030
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:${PORT}/health || exit 1
+
+# Run the specified service
+CMD /app/${SERVICE}
--- a/backend/deploy/docker/Dockerfile.api-gateway
+++ b/backend/deploy/docker/Dockerfile.api-gateway
@@ -0,0 +1,32 @@
+# Build stage
+FROM golang:1.22-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /api-gateway ./cmd/api-gateway
+
+# Runtime stage
+FROM alpine:3.19
+
+RUN apk add --no-cache ca-certificates tzdata
+
+WORKDIR /app
+
+COPY --from=builder /api-gateway .
+
+ENV PORT=3015
+ENV GIN_MODE=release
+
+EXPOSE 3015
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:3015/health || exit 1
+
+CMD ["./api-gateway"]
--- a/backend/deploy/docker/Dockerfile.chat-svc
+++ b/backend/deploy/docker/Dockerfile.chat-svc
@@ -0,0 +1,31 @@
+# Dockerfile for chat-svc only
+FROM golang:1.22-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/chat-svc ./cmd/chat-svc
+
+# Runtime stage
+FROM alpine:3.19
+
+RUN apk add --no-cache ca-certificates tzdata
+
+WORKDIR /app
+
+COPY --from=builder /bin/chat-svc /app/chat-svc
+
+ENV PORT=3005
+
+EXPOSE 3005
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:3005/health || exit 1
+
+CMD ["/app/chat-svc"]
--- a/backend/deploy/docker/Dockerfile.computer-svc
+++ b/backend/deploy/docker/Dockerfile.computer-svc
@@ -0,0 +1,32 @@
+# Dockerfile for computer-svc only
+FROM golang:1.22-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+# Build only computer-svc
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/computer-svc ./cmd/computer-svc
+
+# Runtime stage
+FROM alpine:3.19
+
+RUN apk add --no-cache ca-certificates tzdata docker-cli
+
+WORKDIR /app
+
+COPY --from=builder /bin/computer-svc /app/computer-svc
+
+ENV PORT=3030
+
+EXPOSE 3030
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:3030/health || exit 1
+
+CMD ["/app/computer-svc"]
--- a/backend/deploy/docker/Dockerfile.discover-svc
+++ b/backend/deploy/docker/Dockerfile.discover-svc
@@ -0,0 +1,31 @@
+# Dockerfile for discover-svc only
+FROM golang:1.22-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/discover-svc ./cmd/discover-svc
+
+# Runtime stage
+FROM alpine:3.19
+
+RUN apk add --no-cache ca-certificates tzdata
+
+WORKDIR /app
+
+COPY --from=builder /bin/discover-svc /app/discover-svc
+
+ENV PORT=3002
+
+EXPOSE 3002
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:3002/health || exit 1
+
+CMD ["/app/discover-svc"]
--- a/backend/deploy/docker/Dockerfile.search-svc
+++ b/backend/deploy/docker/Dockerfile.search-svc
@@ -0,0 +1,31 @@
+# Dockerfile for search-svc only
+FROM golang:1.22-alpine AS builder
+
+RUN apk add --no-cache git ca-certificates
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /bin/search-svc ./cmd/search-svc
+
+# Runtime stage
+FROM alpine:3.19
+
+RUN apk add --no-cache ca-certificates tzdata
+
+WORKDIR /app
+
+COPY --from=builder /bin/search-svc /app/search-svc
+
+ENV PORT=3001
+
+EXPOSE 3001
+
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+    CMD wget --no-verbose --tries=1 --spider http://localhost:3001/health || exit 1
+
+CMD ["/app/search-svc"]
--- a/backend/deploy/docker/docker-compose.yml
+++ b/backend/deploy/docker/docker-compose.yml
@@ -0,0 +1,382 @@
+version: '3.8'
+
+services:
+  api-gateway:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=api-gateway
+      - PORT=3015
+      - CHAT_SVC_URL=http://chat-svc:3005
+      - MASTER_AGENTS_SVC_URL=http://agent-svc:3018
+      - SEARCH_SVC_URL=http://search-svc:3001
+      - LLM_SVC_URL=http://llm-svc:3020
+      - SCRAPER_SVC_URL=http://scraper-svc:3021
+      - THREAD_SVC_URL=http://thread-svc:3027
+      - COMPUTER_SVC_URL=http://computer-svc:3030
+      - DISCOVER_SVC_URL=http://discover-svc:3002
+      - FINANCE_HEATMAP_SVC_URL=http://finance-heatmap-svc:3033
+      - LEARNING_SVC_URL=http://learning-svc:3034
+    ports:
+      - "3015:3015"
+    depends_on:
+      - chat-svc
+      - agent-svc
+      - thread-svc
+      - computer-svc
+    networks:
+      - gooseek
+
+  chat-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.chat-svc
+    environment:
+      - SERVICE=chat-svc
+      - PORT=3005
+      - MASTER_AGENTS_SVC_URL=http://agent-svc:3018
+      - DISCOVER_SVC_URL=http://discover-svc:3002
+    ports:
+      - "3005:3005"
+    depends_on:
+      - agent-svc
+      - discover-svc
+    networks:
+      - gooseek
+
+  agent-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.agent-svc
+    environment:
+      - SERVICE=agent-svc
+      - PORT=3018
+      - SEARXNG_URL=http://searxng:8080
+      - DISCOVER_SVC_URL=http://discover-svc:3002
+      - CRAWL4AI_URL=http://crawl4ai:11235
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - GEMINI_API_KEY=${GEMINI_API_KEY}
+      - TIMEWEB_API_BASE_URL=${TIMEWEB_API_BASE_URL}
+      - TIMEWEB_AGENT_ACCESS_ID=${TIMEWEB_AGENT_ACCESS_ID}
+      - TIMEWEB_API_KEY=${TIMEWEB_API_KEY}
+    ports:
+      - "3018:3018"
+    depends_on:
+      - search-svc
+      - discover-svc
+    networks:
+      - gooseek
+
+  search-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.search-svc
+    environment:
+      - SERVICE=search-svc
+      - PORT=3001
+      - SEARXNG_URL=http://searxng:8080
+    ports:
+      - "3001:3001"
+    depends_on:
+      - searxng
+    networks:
+      - gooseek
+
+  llm-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=llm-svc
+      - PORT=3020
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - GEMINI_API_KEY=${GEMINI_API_KEY}
+    ports:
+      - "3020:3020"
+    networks:
+      - gooseek
+
+  scraper-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=scraper-svc
+      - PORT=3021
+      - CRAWL4AI_URL=http://crawl4ai:11235
+    ports:
+      - "3021:3021"
+    networks:
+      - gooseek
+
+  discover-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.discover-svc
+    environment:
+      - SERVICE=discover-svc
+      - PORT=3002
+      - SEARXNG_URL=http://searxng:8080
+      - REDIS_URL=redis://redis:6379
+      - DATABASE_URL=postgres://gooseek:gooseek@postgres:5432/gooseek?sslmode=disable
+    ports:
+      - "3002:3002"
+    depends_on:
+      - searxng
+      - postgres
+      - redis
+    networks:
+      - gooseek
+
+  collection-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=collection-svc
+      - PORT=3025
+      - DATABASE_URL=postgres://gooseek:gooseek@postgres:5432/gooseek?sslmode=disable
+      - JWT_SECRET=${JWT_SECRET}
+      - AUTH_SVC_URL=${AUTH_SVC_URL}
+    ports:
+      - "3025:3025"
+    depends_on:
+      - postgres
+    networks:
+      - gooseek
+
+  file-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=file-svc
+      - PORT=3026
+      - DATABASE_URL=postgres://gooseek:gooseek@postgres:5432/gooseek?sslmode=disable
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - JWT_SECRET=${JWT_SECRET}
+      - FILE_STORAGE_PATH=/data/files
+    ports:
+      - "3026:3026"
+    volumes:
+      - file-storage:/data/files
+    depends_on:
+      - postgres
+    networks:
+      - gooseek
+
+  thread-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=thread-svc
+      - PORT=3027
+      - DATABASE_URL=postgres://gooseek:gooseek@postgres:5432/gooseek?sslmode=disable
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - JWT_SECRET=${JWT_SECRET}
+      - AUTH_SVC_URL=${AUTH_SVC_URL}
+    ports:
+      - "3027:3027"
+    depends_on:
+      - postgres
+    networks:
+      - gooseek
+
+  computer-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=computer-svc
+      - PORT=3030
+      - COMPUTER_SVC_PORT=3030
+      - DATABASE_URL=postgres://gooseek:gooseek@postgres:5432/gooseek?sslmode=disable
+      - REDIS_URL=redis://redis:6379
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - GEMINI_API_KEY=${GEMINI_API_KEY}
+      - TELEGRAM_BOT_TOKEN=${TELEGRAM_BOT_TOKEN}
+      - SANDBOX_IMAGE=python:3.11-slim
+      - BROWSER_SVC_URL=http://browser-svc:3050
+    ports:
+      - "3030:3030"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - checkpoints:/data/checkpoints
+    depends_on:
+      - postgres
+      - redis
+      - browser-svc
+    networks:
+      - gooseek
+
+  browser-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=browser-svc
+      - PORT=3050
+      - BROWSER_SVC_PORT=3050
+    ports:
+      - "3050:3050"
+    volumes:
+      - screenshots:/tmp/gooseek-screenshots
+      - recordings:/tmp/gooseek-recordings
+    networks:
+      - gooseek
+
+  labs-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=labs-svc
+      - PORT=3031
+      - LABS_SVC_PORT=3031
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+    ports:
+      - "3031:3031"
+    networks:
+      - gooseek
+
+  podcast-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=podcast-svc
+      - PORT=3032
+      - PODCAST_SVC_PORT=3032
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - ELEVENLABS_API_KEY=${ELEVENLABS_API_KEY}
+    ports:
+      - "3032:3032"
+    volumes:
+      - podcasts:/data/podcasts
+    networks:
+      - gooseek
+
+  finance-heatmap-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=finance-heatmap-svc
+      - PORT=3033
+      - REDIS_URL=redis://redis:6379
+    ports:
+      - "3033:3033"
+    depends_on:
+      - redis
+    networks:
+      - gooseek
+
+  learning-svc:
+    build:
+      context: ../..
+      dockerfile: deploy/docker/Dockerfile.all
+    environment:
+      - SERVICE=learning-svc
+      - PORT=3034
+      - LEARNING_SVC_PORT=3034
+      - TIMEWEB_API_BASE_URL=${TIMEWEB_API_BASE_URL}
+      - TIMEWEB_AGENT_ACCESS_ID=${TIMEWEB_AGENT_ACCESS_ID}
+      - TIMEWEB_API_KEY=${TIMEWEB_API_KEY}
+      - DEFAULT_LLM_MODEL=${DEFAULT_LLM_MODEL:-gpt-4o-mini}
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
+      - DATABASE_URL=postgres://gooseek:gooseek@postgres:5432/gooseek?sslmode=disable
+    ports:
+      - "3034:3034"
+    depends_on:
+      - postgres
+    networks:
+      - gooseek
+
+  webui:
+    build:
+      context: ../../webui
+      dockerfile: Dockerfile
+    environment:
+      - NODE_ENV=production
+      - API_URL=http://api-gateway:3015
+      - NEXT_PUBLIC_API_URL=
+    ports:
+      - "3000:3000"
+    depends_on:
+      - api-gateway
+    networks:
+      - gooseek
+    restart: unless-stopped
+
+  postgres:
+    image: postgres:16-alpine
+    environment:
+      - POSTGRES_USER=gooseek
+      - POSTGRES_PASSWORD=gooseek
+      - POSTGRES_DB=gooseek
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U gooseek"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - gooseek
+
+  redis:
+    image: redis:7-alpine
+    command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru
+    volumes:
+      - redis-data:/data
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    networks:
+      - gooseek
+
+  searxng:
+    image: searxng/searxng:latest
+    volumes:
+      - ../../../deploy/docker/searxng:/etc/searxng:ro
+    environment:
+      - SEARXNG_BASE_URL=http://localhost:8080
+    ports:
+      - "8080:8080"
+    networks:
+      - gooseek
+
+  crawl4ai:
+    image: unclecode/crawl4ai:latest
+    ports:
+      - "11235:11235"
+    networks:
+      - gooseek
+
+networks:
+  gooseek:
+    driver: bridge
+
+volumes:
+  postgres-data:
+  redis-data:
+  file-storage:
+  checkpoints:
+  screenshots:
+  recordings:
+  podcasts: