fix: back off preview on ffmpeg kill
This commit is contained in:
@@ -256,7 +256,9 @@ func (h *Handler) HandleProxyPreviewMap(w http.ResponseWriter, r *http.Request)
|
|||||||
Duration: duration,
|
Duration: duration,
|
||||||
})
|
})
|
||||||
if previewErr != nil {
|
if previewErr != nil {
|
||||||
log.Printf("preview map error mal_id=%d ep=%s mode=%s: %v", malID, episode, mode, previewErr)
|
if previewErr.Error() != "preview temporarily disabled" {
|
||||||
|
log.Printf("preview map error mal_id=%d ep=%s mode=%s: %v", malID, episode, mode, previewErr)
|
||||||
|
}
|
||||||
http.Error(w, "failed to generate preview map", http.StatusBadGateway)
|
http.Error(w, "failed to generate preview map", http.StatusBadGateway)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"crypto/sha1"
|
"crypto/sha1"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math"
|
"math"
|
||||||
@@ -26,6 +27,7 @@ const (
|
|||||||
previewFrameInterval = 10
|
previewFrameInterval = 10
|
||||||
previewFrameLimit = previewGridColumns * previewGridRows
|
previewFrameLimit = previewGridColumns * previewGridRows
|
||||||
previewGenerationLimit = 120 * time.Second
|
previewGenerationLimit = 120 * time.Second
|
||||||
|
previewFailureTTL = 2 * time.Minute
|
||||||
previewManifestName = "map.json"
|
previewManifestName = "map.json"
|
||||||
spriteFileName = "sprite.jpg"
|
spriteFileName = "sprite.jpg"
|
||||||
)
|
)
|
||||||
@@ -67,6 +69,9 @@ func (s *Service) EnsurePreviewMap(ctx context.Context, req PreviewRequest) (Pre
|
|||||||
|
|
||||||
previewHash := hashPreviewIdentity(req.MalID, normalizedEpisode, normalizedMode, req.Source, req.Referer)
|
previewHash := hashPreviewIdentity(req.MalID, normalizedEpisode, normalizedMode, req.Source, req.Referer)
|
||||||
previewKey := fmt.Sprintf("%d-%s", req.MalID, previewHash)
|
previewKey := fmt.Sprintf("%d-%s", req.MalID, previewHash)
|
||||||
|
if s.previewFailureActive(previewKey) {
|
||||||
|
return PreviewMap{}, "", fmt.Errorf("preview temporarily disabled")
|
||||||
|
}
|
||||||
previewDir := filepath.Join(s.previewRoot, previewKey)
|
previewDir := filepath.Join(s.previewRoot, previewKey)
|
||||||
manifestPath := filepath.Join(previewDir, previewManifestName)
|
manifestPath := filepath.Join(previewDir, previewManifestName)
|
||||||
spritePath := filepath.Join(previewDir, spriteFileName)
|
spritePath := filepath.Join(previewDir, spriteFileName)
|
||||||
@@ -120,8 +125,12 @@ func (s *Service) EnsurePreviewMap(ctx context.Context, req PreviewRequest) (Pre
|
|||||||
interval := selectPreviewInterval(duration)
|
interval := selectPreviewInterval(duration)
|
||||||
|
|
||||||
if err := generatePreviewSprite(ctxWithTimeout, ffmpegPath, req.Source, req.Referer, spritePath, interval); err != nil {
|
if err := generatePreviewSprite(ctxWithTimeout, ffmpegPath, req.Source, req.Referer, spritePath, interval); err != nil {
|
||||||
|
if shouldBackoffPreview(err) {
|
||||||
|
s.markPreviewFailure(previewKey)
|
||||||
|
}
|
||||||
return PreviewMap{}, "", err
|
return PreviewMap{}, "", err
|
||||||
}
|
}
|
||||||
|
s.clearPreviewFailure(previewKey)
|
||||||
|
|
||||||
mapData := buildPreviewMap(duration, interval)
|
mapData := buildPreviewMap(duration, interval)
|
||||||
if err := writePreviewManifest(manifestPath, mapData); err != nil {
|
if err := writePreviewManifest(manifestPath, mapData); err != nil {
|
||||||
@@ -159,6 +168,37 @@ func (s *Service) previewLock(key string) *sync.Mutex {
|
|||||||
return newLock
|
return newLock
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) previewFailureActive(key string) bool {
|
||||||
|
s.previewFailMu.Lock()
|
||||||
|
defer s.previewFailMu.Unlock()
|
||||||
|
|
||||||
|
expiresAt, ok := s.previewFailTTL[key]
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if time.Now().After(expiresAt) {
|
||||||
|
delete(s.previewFailTTL, key)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) markPreviewFailure(key string) {
|
||||||
|
s.previewFailMu.Lock()
|
||||||
|
defer s.previewFailMu.Unlock()
|
||||||
|
|
||||||
|
s.previewFailTTL[key] = time.Now().Add(previewFailureTTL)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) clearPreviewFailure(key string) {
|
||||||
|
s.previewFailMu.Lock()
|
||||||
|
defer s.previewFailMu.Unlock()
|
||||||
|
|
||||||
|
delete(s.previewFailTTL, key)
|
||||||
|
}
|
||||||
|
|
||||||
func hashPreviewIdentity(malID int, episode string, mode string, source string, referer string) string {
|
func hashPreviewIdentity(malID int, episode string, mode string, source string, referer string) string {
|
||||||
payload := fmt.Sprintf("%d|%s|%s|%s|%s", malID, episode, mode, source, referer)
|
payload := fmt.Sprintf("%d|%s|%s|%s|%s", malID, episode, mode, source, referer)
|
||||||
sum := sha1.Sum([]byte(payload))
|
sum := sha1.Sum([]byte(payload))
|
||||||
@@ -398,3 +438,26 @@ func isFinitePositive(value float64) bool {
|
|||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func shouldBackoffPreview(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
errText := strings.ToLower(err.Error())
|
||||||
|
if strings.Contains(errText, "signal: killed") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if strings.Contains(errText, "context canceled") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if strings.Contains(errText, "cannot allocate memory") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
@@ -28,6 +28,8 @@ type Service struct {
|
|||||||
previewRoot string
|
previewRoot string
|
||||||
previewMu sync.Mutex
|
previewMu sync.Mutex
|
||||||
previewLocks map[string]*sync.Mutex
|
previewLocks map[string]*sync.Mutex
|
||||||
|
previewFailMu sync.Mutex
|
||||||
|
previewFailTTL map[string]time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
type sourceScore struct {
|
type sourceScore struct {
|
||||||
@@ -47,6 +49,7 @@ func NewService(jikanClient *jikan.Client, db database.Querier) *Service {
|
|||||||
db: db,
|
db: db,
|
||||||
previewRoot: newPreviewRootDir(),
|
previewRoot: newPreviewRootDir(),
|
||||||
previewLocks: make(map[string]*sync.Mutex),
|
previewLocks: make(map[string]*sync.Mutex),
|
||||||
|
previewFailTTL: make(map[string]time.Time),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user