feat: Go backend, enhanced search, new widgets, Docker deploy
Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
This commit is contained in:
home
432
backend/internal/computer/connectors/storage.go
Normal file
432
backend/internal/computer/connectors/storage.go
Normal file
@@ -0,0 +1,432 @@
|
||||
package connectors
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type StorageConfig struct {
|
||||
Endpoint string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
BucketName string
|
||||
UseSSL bool
|
||||
Region string
|
||||
PublicURL string
|
||||
}
|
||||
|
||||
type StorageConnector struct {
|
||||
cfg StorageConfig
|
||||
client *minio.Client
|
||||
}
|
||||
|
||||
func NewStorageConnector(cfg StorageConfig) (*StorageConnector, error) {
|
||||
client, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: cfg.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create storage client: %w", err)
|
||||
}
|
||||
|
||||
return &StorageConnector{
|
||||
cfg: cfg,
|
||||
client: client,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) ID() string {
|
||||
return "storage"
|
||||
}
|
||||
|
||||
func (s *StorageConnector) Name() string {
|
||||
return "Storage"
|
||||
}
|
||||
|
||||
func (s *StorageConnector) Description() string {
|
||||
return "Store and retrieve files from S3-compatible storage"
|
||||
}
|
||||
|
||||
func (s *StorageConnector) GetActions() []Action {
|
||||
return []Action{
|
||||
{
|
||||
Name: "upload",
|
||||
Description: "Upload a file",
|
||||
Schema: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
||||
"content": map[string]interface{}{"type": "string", "description": "File content (base64 or text)"},
|
||||
"content_type": map[string]interface{}{"type": "string", "description": "MIME type"},
|
||||
"public": map[string]interface{}{"type": "boolean", "description": "Make file publicly accessible"},
|
||||
},
|
||||
},
|
||||
Required: []string{"path", "content"},
|
||||
},
|
||||
{
|
||||
Name: "download",
|
||||
Description: "Download a file",
|
||||
Schema: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
||||
},
|
||||
},
|
||||
Required: []string{"path"},
|
||||
},
|
||||
{
|
||||
Name: "delete",
|
||||
Description: "Delete a file",
|
||||
Schema: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
||||
},
|
||||
},
|
||||
Required: []string{"path"},
|
||||
},
|
||||
{
|
||||
Name: "list",
|
||||
Description: "List files in a directory",
|
||||
Schema: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"prefix": map[string]interface{}{"type": "string", "description": "Path prefix"},
|
||||
"limit": map[string]interface{}{"type": "integer", "description": "Max results"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "get_url",
|
||||
Description: "Get a presigned URL for a file",
|
||||
Schema: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
||||
"expires": map[string]interface{}{"type": "integer", "description": "URL expiry in seconds"},
|
||||
},
|
||||
},
|
||||
Required: []string{"path"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StorageConnector) Validate(params map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) Execute(ctx context.Context, action string, params map[string]interface{}) (interface{}, error) {
|
||||
switch action {
|
||||
case "upload":
|
||||
return s.upload(ctx, params)
|
||||
case "download":
|
||||
return s.download(ctx, params)
|
||||
case "delete":
|
||||
return s.deleteFile(ctx, params)
|
||||
case "list":
|
||||
return s.list(ctx, params)
|
||||
case "get_url":
|
||||
return s.getURL(ctx, params)
|
||||
default:
|
||||
return nil, errors.New("unknown action: " + action)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StorageConnector) upload(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
||||
path := params["path"].(string)
|
||||
content := params["content"].(string)
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
if ct, ok := params["content_type"].(string); ok {
|
||||
contentType = ct
|
||||
}
|
||||
|
||||
if contentType == "" {
|
||||
contentType = s.detectContentType(path)
|
||||
}
|
||||
|
||||
reader := bytes.NewReader([]byte(content))
|
||||
size := int64(len(content))
|
||||
|
||||
info, err := s.client.PutObject(ctx, s.cfg.BucketName, path, reader, size, minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("upload failed: %w", err)
|
||||
}
|
||||
|
||||
url := ""
|
||||
if s.cfg.PublicURL != "" {
|
||||
url = fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s.cfg.PublicURL, "/"), s.cfg.BucketName, path)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"success": true,
|
||||
"path": path,
|
||||
"size": info.Size,
|
||||
"etag": info.ETag,
|
||||
"url": url,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) UploadBytes(ctx context.Context, path string, content []byte, contentType string) (string, error) {
|
||||
if contentType == "" {
|
||||
contentType = s.detectContentType(path)
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(content)
|
||||
size := int64(len(content))
|
||||
|
||||
_, err := s.client.PutObject(ctx, s.cfg.BucketName, path, reader, size, minio.PutObjectOptions{
|
||||
ContentType: contentType,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if s.cfg.PublicURL != "" {
|
||||
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s.cfg.PublicURL, "/"), s.cfg.BucketName, path), nil
|
||||
}
|
||||
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) download(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
||||
path := params["path"].(string)
|
||||
|
||||
obj, err := s.client.GetObject(ctx, s.cfg.BucketName, path, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download failed: %w", err)
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
content, err := io.ReadAll(obj)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read failed: %w", err)
|
||||
}
|
||||
|
||||
stat, _ := obj.Stat()
|
||||
|
||||
return map[string]interface{}{
|
||||
"success": true,
|
||||
"path": path,
|
||||
"content": string(content),
|
||||
"size": len(content),
|
||||
"content_type": stat.ContentType,
|
||||
"modified": stat.LastModified,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) DownloadBytes(ctx context.Context, path string) ([]byte, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.cfg.BucketName, path, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer obj.Close()
|
||||
|
||||
return io.ReadAll(obj)
|
||||
}
|
||||
|
||||
func (s *StorageConnector) deleteFile(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
||||
path := params["path"].(string)
|
||||
|
||||
err := s.client.RemoveObject(ctx, s.cfg.BucketName, path, minio.RemoveObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("delete failed: %w", err)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"success": true,
|
||||
"path": path,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) list(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
||||
prefix := ""
|
||||
if p, ok := params["prefix"].(string); ok {
|
||||
prefix = p
|
||||
}
|
||||
|
||||
limit := 100
|
||||
if l, ok := params["limit"].(float64); ok {
|
||||
limit = int(l)
|
||||
}
|
||||
|
||||
objects := s.client.ListObjects(ctx, s.cfg.BucketName, minio.ListObjectsOptions{
|
||||
Prefix: prefix,
|
||||
Recursive: true,
|
||||
})
|
||||
|
||||
var files []map[string]interface{}
|
||||
count := 0
|
||||
|
||||
for obj := range objects {
|
||||
if obj.Err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
files = append(files, map[string]interface{}{
|
||||
"path": obj.Key,
|
||||
"size": obj.Size,
|
||||
"modified": obj.LastModified,
|
||||
"etag": obj.ETag,
|
||||
})
|
||||
|
||||
count++
|
||||
if count >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"success": true,
|
||||
"files": files,
|
||||
"count": len(files),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) getURL(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
||||
path := params["path"].(string)
|
||||
|
||||
expires := 3600
|
||||
if e, ok := params["expires"].(float64); ok {
|
||||
expires = int(e)
|
||||
}
|
||||
|
||||
url, err := s.client.PresignedGetObject(ctx, s.cfg.BucketName, path, time.Duration(expires)*time.Second, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate URL: %w", err)
|
||||
}
|
||||
|
||||
return map[string]interface{}{
|
||||
"success": true,
|
||||
"url": url.String(),
|
||||
"expires": expires,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *StorageConnector) GetPublicURL(path string) string {
|
||||
if s.cfg.PublicURL != "" {
|
||||
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s.cfg.PublicURL, "/"), s.cfg.BucketName, path)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *StorageConnector) detectContentType(path string) string {
|
||||
ext := strings.ToLower(filepath.Ext(path))
|
||||
|
||||
contentTypes := map[string]string{
|
||||
".html": "text/html",
|
||||
".css": "text/css",
|
||||
".js": "application/javascript",
|
||||
".json": "application/json",
|
||||
".xml": "application/xml",
|
||||
".pdf": "application/pdf",
|
||||
".zip": "application/zip",
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".svg": "image/svg+xml",
|
||||
".mp4": "video/mp4",
|
||||
".mp3": "audio/mpeg",
|
||||
".txt": "text/plain",
|
||||
".md": "text/markdown",
|
||||
".csv": "text/csv",
|
||||
".py": "text/x-python",
|
||||
".go": "text/x-go",
|
||||
".rs": "text/x-rust",
|
||||
}
|
||||
|
||||
if ct, ok := contentTypes[ext]; ok {
|
||||
return ct
|
||||
}
|
||||
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (s *StorageConnector) EnsureBucket(ctx context.Context) error {
|
||||
exists, err := s.client.BucketExists(ctx, s.cfg.BucketName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !exists {
|
||||
return s.client.MakeBucket(ctx, s.cfg.BucketName, minio.MakeBucketOptions{
|
||||
Region: s.cfg.Region,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewLocalStorageConnector(basePath string) *LocalStorageConnector {
|
||||
return &LocalStorageConnector{basePath: basePath}
|
||||
}
|
||||
|
||||
type LocalStorageConnector struct {
|
||||
basePath string
|
||||
}
|
||||
|
||||
func (l *LocalStorageConnector) ID() string {
|
||||
return "local_storage"
|
||||
}
|
||||
|
||||
func (l *LocalStorageConnector) Name() string {
|
||||
return "Local Storage"
|
||||
}
|
||||
|
||||
func (l *LocalStorageConnector) Description() string {
|
||||
return "Store files on local filesystem"
|
||||
}
|
||||
|
||||
func (l *LocalStorageConnector) GetActions() []Action {
|
||||
return []Action{
|
||||
{Name: "upload", Description: "Upload a file"},
|
||||
{Name: "download", Description: "Download a file"},
|
||||
{Name: "delete", Description: "Delete a file"},
|
||||
{Name: "list", Description: "List files"},
|
||||
}
|
||||
}
|
||||
|
||||
func (l *LocalStorageConnector) Validate(params map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *LocalStorageConnector) Execute(ctx context.Context, action string, params map[string]interface{}) (interface{}, error) {
|
||||
switch action {
|
||||
case "upload":
|
||||
path := params["path"].(string)
|
||||
content := params["content"].(string)
|
||||
fullPath := filepath.Join(l.basePath, path)
|
||||
os.MkdirAll(filepath.Dir(fullPath), 0755)
|
||||
err := os.WriteFile(fullPath, []byte(content), 0644)
|
||||
return map[string]interface{}{"success": err == nil, "path": path}, err
|
||||
|
||||
case "download":
|
||||
path := params["path"].(string)
|
||||
fullPath := filepath.Join(l.basePath, path)
|
||||
content, err := os.ReadFile(fullPath)
|
||||
return map[string]interface{}{"success": err == nil, "content": string(content)}, err
|
||||
|
||||
case "delete":
|
||||
path := params["path"].(string)
|
||||
fullPath := filepath.Join(l.basePath, path)
|
||||
err := os.Remove(fullPath)
|
||||
return map[string]interface{}{"success": err == nil}, err
|
||||
|
||||
default:
|
||||
return nil, errors.New("unknown action")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user