Major changes: - Add Go backend (backend/) with microservices architecture - Enhanced master-agents-svc: reranker, content-classifier, stealth-crawler, proxy-manager, media-search, fastClassifier, language detection - New web-svc widgets: KnowledgeCard, ProductCard, ProfileCard, VideoCard, UnifiedCard, CardGallery, InlineImageGallery, SourcesPanel, RelatedQuestions - Improved discover-svc with discover-db integration - Docker deployment improvements (Caddyfile, vendor.sh, BUILD.md) - Library-svc: project_id schema migration - Remove deprecated finance-svc and travel-svc - Localization improvements across services Made-with: Cursor
433 lines
11 KiB
Go
433 lines
11 KiB
Go
package connectors
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
)
|
|
|
|
type StorageConfig struct {
|
|
Endpoint string
|
|
AccessKeyID string
|
|
SecretAccessKey string
|
|
BucketName string
|
|
UseSSL bool
|
|
Region string
|
|
PublicURL string
|
|
}
|
|
|
|
type StorageConnector struct {
|
|
cfg StorageConfig
|
|
client *minio.Client
|
|
}
|
|
|
|
func NewStorageConnector(cfg StorageConfig) (*StorageConnector, error) {
|
|
client, err := minio.New(cfg.Endpoint, &minio.Options{
|
|
Creds: credentials.NewStaticV4(cfg.AccessKeyID, cfg.SecretAccessKey, ""),
|
|
Secure: cfg.UseSSL,
|
|
Region: cfg.Region,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create storage client: %w", err)
|
|
}
|
|
|
|
return &StorageConnector{
|
|
cfg: cfg,
|
|
client: client,
|
|
}, nil
|
|
}
|
|
|
|
func (s *StorageConnector) ID() string {
|
|
return "storage"
|
|
}
|
|
|
|
func (s *StorageConnector) Name() string {
|
|
return "Storage"
|
|
}
|
|
|
|
func (s *StorageConnector) Description() string {
|
|
return "Store and retrieve files from S3-compatible storage"
|
|
}
|
|
|
|
func (s *StorageConnector) GetActions() []Action {
|
|
return []Action{
|
|
{
|
|
Name: "upload",
|
|
Description: "Upload a file",
|
|
Schema: map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
|
"content": map[string]interface{}{"type": "string", "description": "File content (base64 or text)"},
|
|
"content_type": map[string]interface{}{"type": "string", "description": "MIME type"},
|
|
"public": map[string]interface{}{"type": "boolean", "description": "Make file publicly accessible"},
|
|
},
|
|
},
|
|
Required: []string{"path", "content"},
|
|
},
|
|
{
|
|
Name: "download",
|
|
Description: "Download a file",
|
|
Schema: map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
|
},
|
|
},
|
|
Required: []string{"path"},
|
|
},
|
|
{
|
|
Name: "delete",
|
|
Description: "Delete a file",
|
|
Schema: map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
|
},
|
|
},
|
|
Required: []string{"path"},
|
|
},
|
|
{
|
|
Name: "list",
|
|
Description: "List files in a directory",
|
|
Schema: map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"prefix": map[string]interface{}{"type": "string", "description": "Path prefix"},
|
|
"limit": map[string]interface{}{"type": "integer", "description": "Max results"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "get_url",
|
|
Description: "Get a presigned URL for a file",
|
|
Schema: map[string]interface{}{
|
|
"type": "object",
|
|
"properties": map[string]interface{}{
|
|
"path": map[string]interface{}{"type": "string", "description": "Storage path/key"},
|
|
"expires": map[string]interface{}{"type": "integer", "description": "URL expiry in seconds"},
|
|
},
|
|
},
|
|
Required: []string{"path"},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (s *StorageConnector) Validate(params map[string]interface{}) error {
|
|
return nil
|
|
}
|
|
|
|
func (s *StorageConnector) Execute(ctx context.Context, action string, params map[string]interface{}) (interface{}, error) {
|
|
switch action {
|
|
case "upload":
|
|
return s.upload(ctx, params)
|
|
case "download":
|
|
return s.download(ctx, params)
|
|
case "delete":
|
|
return s.deleteFile(ctx, params)
|
|
case "list":
|
|
return s.list(ctx, params)
|
|
case "get_url":
|
|
return s.getURL(ctx, params)
|
|
default:
|
|
return nil, errors.New("unknown action: " + action)
|
|
}
|
|
}
|
|
|
|
func (s *StorageConnector) upload(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
|
path := params["path"].(string)
|
|
content := params["content"].(string)
|
|
|
|
contentType := "application/octet-stream"
|
|
if ct, ok := params["content_type"].(string); ok {
|
|
contentType = ct
|
|
}
|
|
|
|
if contentType == "" {
|
|
contentType = s.detectContentType(path)
|
|
}
|
|
|
|
reader := bytes.NewReader([]byte(content))
|
|
size := int64(len(content))
|
|
|
|
info, err := s.client.PutObject(ctx, s.cfg.BucketName, path, reader, size, minio.PutObjectOptions{
|
|
ContentType: contentType,
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("upload failed: %w", err)
|
|
}
|
|
|
|
url := ""
|
|
if s.cfg.PublicURL != "" {
|
|
url = fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s.cfg.PublicURL, "/"), s.cfg.BucketName, path)
|
|
}
|
|
|
|
return map[string]interface{}{
|
|
"success": true,
|
|
"path": path,
|
|
"size": info.Size,
|
|
"etag": info.ETag,
|
|
"url": url,
|
|
}, nil
|
|
}
|
|
|
|
func (s *StorageConnector) UploadBytes(ctx context.Context, path string, content []byte, contentType string) (string, error) {
|
|
if contentType == "" {
|
|
contentType = s.detectContentType(path)
|
|
}
|
|
|
|
reader := bytes.NewReader(content)
|
|
size := int64(len(content))
|
|
|
|
_, err := s.client.PutObject(ctx, s.cfg.BucketName, path, reader, size, minio.PutObjectOptions{
|
|
ContentType: contentType,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
if s.cfg.PublicURL != "" {
|
|
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s.cfg.PublicURL, "/"), s.cfg.BucketName, path), nil
|
|
}
|
|
|
|
return path, nil
|
|
}
|
|
|
|
func (s *StorageConnector) download(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
|
path := params["path"].(string)
|
|
|
|
obj, err := s.client.GetObject(ctx, s.cfg.BucketName, path, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("download failed: %w", err)
|
|
}
|
|
defer obj.Close()
|
|
|
|
content, err := io.ReadAll(obj)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("read failed: %w", err)
|
|
}
|
|
|
|
stat, _ := obj.Stat()
|
|
|
|
return map[string]interface{}{
|
|
"success": true,
|
|
"path": path,
|
|
"content": string(content),
|
|
"size": len(content),
|
|
"content_type": stat.ContentType,
|
|
"modified": stat.LastModified,
|
|
}, nil
|
|
}
|
|
|
|
func (s *StorageConnector) DownloadBytes(ctx context.Context, path string) ([]byte, error) {
|
|
obj, err := s.client.GetObject(ctx, s.cfg.BucketName, path, minio.GetObjectOptions{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer obj.Close()
|
|
|
|
return io.ReadAll(obj)
|
|
}
|
|
|
|
func (s *StorageConnector) deleteFile(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
|
path := params["path"].(string)
|
|
|
|
err := s.client.RemoveObject(ctx, s.cfg.BucketName, path, minio.RemoveObjectOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("delete failed: %w", err)
|
|
}
|
|
|
|
return map[string]interface{}{
|
|
"success": true,
|
|
"path": path,
|
|
}, nil
|
|
}
|
|
|
|
func (s *StorageConnector) list(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
|
prefix := ""
|
|
if p, ok := params["prefix"].(string); ok {
|
|
prefix = p
|
|
}
|
|
|
|
limit := 100
|
|
if l, ok := params["limit"].(float64); ok {
|
|
limit = int(l)
|
|
}
|
|
|
|
objects := s.client.ListObjects(ctx, s.cfg.BucketName, minio.ListObjectsOptions{
|
|
Prefix: prefix,
|
|
Recursive: true,
|
|
})
|
|
|
|
var files []map[string]interface{}
|
|
count := 0
|
|
|
|
for obj := range objects {
|
|
if obj.Err != nil {
|
|
continue
|
|
}
|
|
|
|
files = append(files, map[string]interface{}{
|
|
"path": obj.Key,
|
|
"size": obj.Size,
|
|
"modified": obj.LastModified,
|
|
"etag": obj.ETag,
|
|
})
|
|
|
|
count++
|
|
if count >= limit {
|
|
break
|
|
}
|
|
}
|
|
|
|
return map[string]interface{}{
|
|
"success": true,
|
|
"files": files,
|
|
"count": len(files),
|
|
}, nil
|
|
}
|
|
|
|
func (s *StorageConnector) getURL(ctx context.Context, params map[string]interface{}) (interface{}, error) {
|
|
path := params["path"].(string)
|
|
|
|
expires := 3600
|
|
if e, ok := params["expires"].(float64); ok {
|
|
expires = int(e)
|
|
}
|
|
|
|
url, err := s.client.PresignedGetObject(ctx, s.cfg.BucketName, path, time.Duration(expires)*time.Second, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to generate URL: %w", err)
|
|
}
|
|
|
|
return map[string]interface{}{
|
|
"success": true,
|
|
"url": url.String(),
|
|
"expires": expires,
|
|
}, nil
|
|
}
|
|
|
|
func (s *StorageConnector) GetPublicURL(path string) string {
|
|
if s.cfg.PublicURL != "" {
|
|
return fmt.Sprintf("%s/%s/%s", strings.TrimSuffix(s.cfg.PublicURL, "/"), s.cfg.BucketName, path)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (s *StorageConnector) detectContentType(path string) string {
|
|
ext := strings.ToLower(filepath.Ext(path))
|
|
|
|
contentTypes := map[string]string{
|
|
".html": "text/html",
|
|
".css": "text/css",
|
|
".js": "application/javascript",
|
|
".json": "application/json",
|
|
".xml": "application/xml",
|
|
".pdf": "application/pdf",
|
|
".zip": "application/zip",
|
|
".png": "image/png",
|
|
".jpg": "image/jpeg",
|
|
".jpeg": "image/jpeg",
|
|
".gif": "image/gif",
|
|
".svg": "image/svg+xml",
|
|
".mp4": "video/mp4",
|
|
".mp3": "audio/mpeg",
|
|
".txt": "text/plain",
|
|
".md": "text/markdown",
|
|
".csv": "text/csv",
|
|
".py": "text/x-python",
|
|
".go": "text/x-go",
|
|
".rs": "text/x-rust",
|
|
}
|
|
|
|
if ct, ok := contentTypes[ext]; ok {
|
|
return ct
|
|
}
|
|
|
|
return "application/octet-stream"
|
|
}
|
|
|
|
func (s *StorageConnector) EnsureBucket(ctx context.Context) error {
|
|
exists, err := s.client.BucketExists(ctx, s.cfg.BucketName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if !exists {
|
|
return s.client.MakeBucket(ctx, s.cfg.BucketName, minio.MakeBucketOptions{
|
|
Region: s.cfg.Region,
|
|
})
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func NewLocalStorageConnector(basePath string) *LocalStorageConnector {
|
|
return &LocalStorageConnector{basePath: basePath}
|
|
}
|
|
|
|
type LocalStorageConnector struct {
|
|
basePath string
|
|
}
|
|
|
|
func (l *LocalStorageConnector) ID() string {
|
|
return "local_storage"
|
|
}
|
|
|
|
func (l *LocalStorageConnector) Name() string {
|
|
return "Local Storage"
|
|
}
|
|
|
|
func (l *LocalStorageConnector) Description() string {
|
|
return "Store files on local filesystem"
|
|
}
|
|
|
|
func (l *LocalStorageConnector) GetActions() []Action {
|
|
return []Action{
|
|
{Name: "upload", Description: "Upload a file"},
|
|
{Name: "download", Description: "Download a file"},
|
|
{Name: "delete", Description: "Delete a file"},
|
|
{Name: "list", Description: "List files"},
|
|
}
|
|
}
|
|
|
|
func (l *LocalStorageConnector) Validate(params map[string]interface{}) error {
|
|
return nil
|
|
}
|
|
|
|
func (l *LocalStorageConnector) Execute(ctx context.Context, action string, params map[string]interface{}) (interface{}, error) {
|
|
switch action {
|
|
case "upload":
|
|
path := params["path"].(string)
|
|
content := params["content"].(string)
|
|
fullPath := filepath.Join(l.basePath, path)
|
|
os.MkdirAll(filepath.Dir(fullPath), 0755)
|
|
err := os.WriteFile(fullPath, []byte(content), 0644)
|
|
return map[string]interface{}{"success": err == nil, "path": path}, err
|
|
|
|
case "download":
|
|
path := params["path"].(string)
|
|
fullPath := filepath.Join(l.basePath, path)
|
|
content, err := os.ReadFile(fullPath)
|
|
return map[string]interface{}{"success": err == nil, "content": string(content)}, err
|
|
|
|
case "delete":
|
|
path := params["path"].(string)
|
|
fullPath := filepath.Join(l.basePath, path)
|
|
err := os.Remove(fullPath)
|
|
return map[string]interface{}{"success": err == nil}, err
|
|
|
|
default:
|
|
return nil, errors.New("unknown action")
|
|
}
|
|
}
|