simplify: remove complex episode fetching, use simple sequential pagination

This commit is contained in:
2026-05-03 14:22:58 +02:00
committed by Mikkel Elvers
parent 4ba4e51d95
commit f2afa6a4e5
3 changed files with 80 additions and 391 deletions

View File

@@ -11,8 +11,6 @@ import (
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
"sync"
"time"
"mal/integrations/jikan" "mal/integrations/jikan"
database "mal/internal/db" database "mal/internal/db"
@@ -58,34 +56,19 @@ func (h *Handler) HandleWatchPage(w http.ResponseWriter, r *http.Request) {
return return
} }
// Get essential episodes (first and last pages) // Fetch episodes sequentially (pages are in correct order: 1-100, 101-200, etc.)
allEpisodes, err := h.jikanClient.GetAllEpisodes(r.Context(), id) pageSize := 100
if err != nil { var allEpisodes []jikan.Episode
log.Printf("watch error fetching episodes: %v", err) for page := 1; ; page++ {
} resp, err := h.jikanClient.GetEpisodes(r.Context(), id, page)
if err != nil || len(resp.Data) == 0 {
// Fetch any metadata overlays (thumbnails) break
videoEpisodes, _ := h.jikanClient.GetVideoEpisodes(r.Context(), id, 1)
videoMeta := make(map[int]jikan.Episode)
for _, ve := range videoEpisodes.Data {
videoMeta[ve.MalID] = ve
}
for i, ep := range allEpisodes {
if ve, ok := videoMeta[ep.MalID]; ok {
if ve.Images != nil && ve.Images.Jpg.ImageURL != "" {
allEpisodes[i].Images = ve.Images
}
} }
} allEpisodes = append(allEpisodes, resp.Data...)
// Deduplicate and prep the list // If we got fewer than pageSize, we've reached the end
seen := make(map[int]bool) if len(resp.Data) < pageSize {
unique := make([]jikan.Episode, 0) break
for _, ep := range allEpisodes {
if !seen[ep.MalID] {
seen[ep.MalID] = true
unique = append(unique, ep)
} }
} }
@@ -127,7 +110,7 @@ func (h *Handler) HandleWatchPage(w http.ResponseWriter, r *http.Request) {
log.Printf("watch data error: %v", err) log.Printf("watch data error: %v", err)
} }
// Update episodes list if fallback has more // Fill gaps with placeholder episodes if fallback has more
if watchData.FallbackEpisodes != nil { if watchData.FallbackEpisodes != nil {
maxCount := 0 maxCount := 0
for _, count := range watchData.FallbackEpisodes { for _, count := range watchData.FallbackEpisodes {
@@ -137,74 +120,34 @@ func (h *Handler) HandleWatchPage(w http.ResponseWriter, r *http.Request) {
} }
epMap := make(map[int]jikan.Episode) epMap := make(map[int]jikan.Episode)
for _, ep := range unique { for _, ep := range allEpisodes {
epMap[ep.MalID] = ep epMap[ep.MalID] = ep
} }
if maxCount > 0 { if maxCount > 0 {
var fullList []jikan.Episode var filled []jikan.Episode
for i := 1; i <= maxCount; i++ { for i := 1; i <= maxCount; i++ {
if ep, ok := epMap[i]; ok { if ep, ok := epMap[i]; ok {
fullList = append(fullList, ep) filled = append(filled, ep)
} else { } else {
fullList = append(fullList, jikan.Episode{ filled = append(filled, jikan.Episode{
MalID: i, MalID: i,
Episode: fmt.Sprintf("Episode %d", i), Episode: fmt.Sprintf("Episode %d", i),
Title: fmt.Sprintf("Episode %d", i), Title: fmt.Sprintf("Episode %d", i),
}) })
} }
} }
unique = fullList allEpisodes = filled
} }
} }
// Update episodes list if fallback has more sort.Slice(allEpisodes, func(i, j int) bool {
if watchData.FallbackEpisodes != nil { return allEpisodes[i].MalID < allEpisodes[j].MalID
maxCount := 0
for _, count := range watchData.FallbackEpisodes {
if count > maxCount {
maxCount = count
}
}
// Ensure we don't have duplicates or missing episodes in the sequence
epMap := make(map[int]jikan.Episode)
for _, ep := range unique {
epMap[ep.MalID] = ep
}
if maxCount > 0 {
var newEpisodes []jikan.Episode
// We build the list from 1 to maxCount to ensure order and completeness
// If we have data from Jikan, we use it. Otherwise we generate a placeholder.
for i := 1; i <= maxCount; i++ {
if ep, ok := epMap[i]; ok {
newEpisodes = append(newEpisodes, ep)
} else {
title := fmt.Sprintf("Episode %d", i)
newEpisodes = append(newEpisodes, jikan.Episode{
MalID: i,
Episode: fmt.Sprintf("Episode %d", i),
Title: title,
Images: &jikan.EpisodeImages{
Jpg: struct {
ImageURL string `json:"image_url"`
}{ImageURL: ""},
},
})
}
}
unique = newEpisodes
}
}
sort.Slice(unique, func(i, j int) bool {
return unique[i].MalID < unique[j].MalID
}) })
if err := templates.GetRenderer().ExecuteTemplate(r.Context(), w, "watch.gohtml", map[string]any{ if err := templates.GetRenderer().ExecuteTemplate(r.Context(), w, "watch.gohtml", map[string]any{
"Anime": anime, "Anime": anime,
"Episodes": unique, "Episodes": allEpisodes,
"WatchData": watchData, "WatchData": watchData,
"User": user, "User": user,
"CurrentPath": r.URL.Path, "CurrentPath": r.URL.Path,
@@ -418,103 +361,54 @@ func (h *Handler) HandleEpisodeThumbnails(w http.ResponseWriter, r *http.Request
return return
} }
// Get essential episodes (first and last pages) // Fetch episodes sequentially
allEpisodes, err := h.jikanClient.GetAllEpisodes(r.Context(), id) pageSize := 100
if err != nil { var allEpisodes []jikan.Episode
http.Error(w, "failed to get episodes", http.StatusInternalServerError) for page := 1; ; page++ {
return resp, err := h.jikanClient.GetEpisodes(r.Context(), id, page)
} if err != nil || len(resp.Data) == 0 {
break
// Also get video episodes for richer metadata (thumbnails) on recent episodes }
videoEpisodes, _ := h.jikanClient.GetVideoEpisodes(r.Context(), id, 1) allEpisodes = append(allEpisodes, resp.Data...)
if len(resp.Data) < pageSize {
// Merge metadata break
videoMeta := make(map[int]jikan.Episode)
for _, ve := range videoEpisodes.Data {
videoMeta[ve.MalID] = ve
}
for i, ep := range allEpisodes {
if ve, ok := videoMeta[ep.MalID]; ok {
if ve.Images != nil && ve.Images.Jpg.ImageURL != "" {
allEpisodes[i].Images = ve.Images
}
} }
} }
// Dedup and sort // Fill gaps if anime has known total
seen := make(map[int]bool)
unique := make([]jikan.Episode, 0, len(allEpisodes))
for _, ep := range allEpisodes {
if !seen[ep.MalID] {
seen[ep.MalID] = true
unique = append(unique, ep)
}
}
// Calculate total count from anime info for complete list
anime, _ := h.jikanClient.GetAnimeByID(r.Context(), id) anime, _ := h.jikanClient.GetAnimeByID(r.Context(), id)
maxCount := anime.Episodes if anime.Episodes > 0 && anime.Episodes > len(allEpisodes) {
epMap := make(map[int]jikan.Episode)
epMap := make(map[int]jikan.Episode) for _, ep := range allEpisodes {
for _, ep := range unique { epMap[ep.MalID] = ep
epMap[ep.MalID] = ep }
} var filled []jikan.Episode
for i := 1; i <= anime.Episodes; i++ {
if maxCount > 0 {
var fullList []jikan.Episode
for i := 1; i <= maxCount; i++ {
if ep, ok := epMap[i]; ok { if ep, ok := epMap[i]; ok {
fullList = append(fullList, ep) filled = append(filled, ep)
} else { } else {
fullList = append(fullList, jikan.Episode{ filled = append(filled, jikan.Episode{
MalID: i, MalID: i,
Episode: fmt.Sprintf("Episode %d", i), Episode: fmt.Sprintf("Episode %d", i),
Title: fmt.Sprintf("Episode %d", i), Title: fmt.Sprintf("Episode %d", i),
}) })
} }
} }
unique = fullList allEpisodes = filled
} }
sort.Slice(unique, func(i, j int) bool { type Result struct {
return unique[i].MalID < unique[j].MalID
})
type ThumbResult struct {
MalID int `json:"mal_id"` MalID int `json:"mal_id"`
URL string `json:"url"` Title string `json:"title"`
Title string `json:"title,omitempty"`
} }
results := make([]ThumbResult, len(unique)) results := make([]Result, len(allEpisodes))
for i, ep := range allEpisodes {
// Use a semaphore to limit concurrent scraping requests to avoid MAL bans results[i] = Result{
sem := make(chan struct{}, 2) MalID: ep.MalID,
var wg sync.WaitGroup Title: ep.Title,
}
for i := range unique {
wg.Add(1)
go func(idx int) {
defer wg.Done()
sem <- struct{}{} // Acquire
// Add a small jittered delay between requests to avoid 405/429
time.Sleep(time.Duration(200+idx%300) * time.Millisecond)
defer func() { <-sem }() // Release
ep := unique[idx]
imgURL := ep.GetFallbackImage(id)
results[idx] = ThumbResult{
MalID: ep.MalID,
URL: imgURL,
Title: ep.Title,
}
}(i)
} }
wg.Wait()
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results) json.NewEncoder(w).Encode(results)

View File

@@ -3,118 +3,9 @@ package jikan
import ( import (
"context" "context"
"fmt" "fmt"
"io"
"log"
"net/http"
"regexp"
"strconv"
"strings"
"time" "time"
) )
const bannedImageURL = "https://myanimelist.net/images/icon-banned-youtube.png"
const placeholderImageURL = "https://myanimelist.net/images/episodes/videos/icon-thumbs-not-available.png"
var httpClient = &http.Client{Timeout: 10 * time.Second}
func (e *Episode) GetFallbackImage(animeID int) string {
imageUrl := ""
if e.Images != nil {
imageUrl = e.Images.Jpg.ImageURL
}
// Determining the episode number reliably. Jikan's Episode string can be "Episode 1" or just "1"
episodeNum := 0
if e.Episode != "" {
re := regexp.MustCompile(`\d+`)
match := re.FindString(e.Episode)
if match != "" {
episodeNum, _ = strconv.Atoi(match)
}
}
// For Video episodes, MalID is often the episode number, but let's check
if episodeNum == 0 {
episodeNum = e.MalID
}
// Always trigger scraping if we encounter the banned icon OR the generic placeholder
// OR if there is no image URL at all
if imageUrl == bannedImageURL || imageUrl == placeholderImageURL || imageUrl == "" {
// MAL URLs follow this format: https://myanimelist.net/anime/20/Naruto/episode/1
// The previous format used /_/ which is sometimes rejected with 405
episodeURL := fmt.Sprintf("https://myanimelist.net/anime/%d/slug/episode/%d", animeID, episodeNum)
fallbackURL := scrapeAnimeImageFromEpisodePage(episodeURL, episodeNum)
if fallbackURL != "" {
return fallbackURL
}
}
return imageUrl
}
func scrapeAnimeImageFromEpisodePage(episodeURL string, episodeNum int) string {
req, err := http.NewRequest("GET", episodeURL, nil)
if err != nil {
return ""
}
// Setting User-Agent is important for MAL
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
resp, err := httpClient.Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
// Log the status code for debugging
if resp.StatusCode != 200 {
log.Printf("[DEBUG] Scraper failed to fetch %s: Status %d", episodeURL, resp.StatusCode)
return ""
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return ""
}
html := string(body)
// MAL sometimes redirects to a URL with a slug.
// We look for the "thumbnail" field in the page source.
// Pattern 1: Look for the specific episode object in the JSON data
episodeStr := strconv.Itoa(episodeNum)
objPattern := regexp.MustCompile(`\{[^{}]*"episode_number":\s*` + episodeStr + `[^{}]*\}`)
match := objPattern.FindString(html)
if match == "" {
// Try a broader search if the strict one fails
objPattern = regexp.MustCompile(`\{[^}]*"episode_number":\s*` + episodeStr + `[^}]*\}`)
match = objPattern.FindString(html)
}
if match != "" {
thumbRe := regexp.MustCompile(`"thumbnail":\s*"([^"]+)"`)
thumbMatch := thumbRe.FindStringSubmatch(match)
if len(thumbMatch) > 1 {
return strings.ReplaceAll(thumbMatch[1], `\/`, `/`)
}
}
// Pattern 2: Fallback to og:image if it's the specific episode page
ogRe := regexp.MustCompile(`<meta\s+property="og:image"\s+content="([^"]+)"`)
ogMatch := ogRe.FindStringSubmatch(html)
if len(ogMatch) > 1 {
// Only use if it looks like an episode thumbnail (contains /episodes/)
if strings.Contains(ogMatch[1], "/episodes/") {
return ogMatch[1]
}
}
return ""
}
func (c *Client) GetEpisodes(ctx context.Context, animeID int, page int) (EpisodesResponse, error) { func (c *Client) GetEpisodes(ctx context.Context, animeID int, page int) (EpisodesResponse, error) {
if page < 1 { if page < 1 {
page = 1 page = 1
@@ -128,86 +19,14 @@ func (c *Client) GetEpisodes(ctx context.Context, animeID int, page int) (Episod
return result, err return result, err
} }
func (c *Client) GetVideoEpisodes(ctx context.Context, animeID int, page int) (EpisodesResponse, error) { func (c *Client) GetEpisodesRange(ctx context.Context, animeID int, startPage, endPage int) ([]Episode, error) {
if page < 1 { var all []Episode
page = 1 for page := startPage; page <= endPage; page++ {
} resp, err := c.GetEpisodes(ctx, animeID, page)
cacheKey := fmt.Sprintf("anime:%d:videos:episodes:%d", animeID, page)
var result EpisodesResponse
reqURL := fmt.Sprintf("%s/anime/%d/videos/episodes?page=%d", c.baseURL, animeID, page)
err := c.getWithCache(ctx, cacheKey, 12*time.Hour, reqURL, &result)
return result, err
}
func (c *Client) GetEpisode(ctx context.Context, animeID int, episode int) (EpisodeResponse, error) {
cacheKey := fmt.Sprintf("anime:%d:episode:%d", animeID, episode)
var result EpisodeResponse
reqURL := fmt.Sprintf("%s/anime/%d/episodes/%d", c.baseURL, animeID, episode)
err := c.getWithCache(ctx, cacheKey, 24*time.Hour, reqURL, &result)
return result, err
}
func (c *Client) GetAllEpisodes(ctx context.Context, animeID int) ([]Episode, error) {
// First fetch the anime to get total episodes count
anime, err := c.GetAnimeByID(ctx, animeID)
if err != nil {
return nil, err
}
totalEpisodes := anime.Episodes
if totalEpisodes <= 0 {
resp, err := c.GetEpisodes(ctx, animeID, 1)
if err != nil { if err != nil {
return nil, err return all, err
} }
return resp.Data, nil all = append(all, resp.Data...)
} }
return all, nil
// Jikan /episodes/video (which has thumbnails) returns ~39-40 per page.
// Jikan /episodes (standard) returns 100 per page.
// Since the user wants to prioritize the metadata-rich video clips if possible,
// we will calculate based on the 100-per-page standard endpoint for the full list,
// but the background logic remains the same: last page to first.
pageSize := 100
lastPage := (totalEpisodes + (pageSize - 1)) / pageSize
var allEpisodes []Episode
// Fetch last page first (to get most recent episodes immediately)
lastResp, err := c.GetEpisodes(ctx, animeID, lastPage)
if err == nil {
allEpisodes = append(allEpisodes, lastResp.Data...)
}
// For the rest, fetch them in reverse order in the background
if lastPage > 1 {
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
// Start from lastPage - 1 and go down to 1
for p := lastPage - 1; p >= 1; p-- {
_, _ = c.GetEpisodes(bgCtx, animeID, p)
// Also pre-fetch the video episodes metadata (39 per page)
// to warm the cache for thumbnails
videoPageSize := 39
vPageStart := ((p-1)*pageSize)/videoPageSize + 1
vPageEnd := (p*pageSize)/videoPageSize + 1
for v := vPageEnd; v >= vPageStart; v-- {
_, _ = c.GetVideoEpisodes(bgCtx, animeID, v)
}
select {
case <-bgCtx.Done():
return
case <-time.After(800 * time.Millisecond):
}
}
}()
}
return allEpisodes, nil
} }

View File

@@ -4,54 +4,30 @@
{{$episodes := .Episodes}} {{$episodes := .Episodes}}
{{$currentEpID := .CurrentEpID}} {{$currentEpID := .CurrentEpID}}
<div class="flex flex-col gap-8 pb-12"> <div class="flex flex-col gap-8 pb-12 lg:flex-row lg:gap-6">
<div id="video-player-container"> <div class="flex-1 min-w-0">
{{template "video_player" dict "WatchData" .WatchData "TotalEpisodes" $anime.Episodes}} <div id="video-player-container">
{{template "video_player" dict "WatchData" .WatchData "TotalEpisodes" $anime.Episodes}}
</div>
</div> </div>
{{if eq (len $episodes) 0}} <div class="w-full lg:w-80 xl:w-96 flex-shrink-0">
<div class="flex flex-col items-center justify-center gap-2 py-24 text-neutral-400"> {{if eq (len $episodes) 0}}
<svg class="h-12 w-12 opacity-30" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M15 10l4.553-2.276A1 1 0 0121 8.618v6.764a1 1 0 01-1.447.894L15 14M5 18h8a2 2 0 002-2V8a2 2 0 00-2-2H5a2 2 0 00-2 2v8a2 2 0 002 2z" /></svg> <div class="flex flex-col items-center justify-center gap-2 py-12 text-neutral-400">
<p class="text-lg">No episodes found for this anime.</p> <svg class="h-10 w-10 opacity-30" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M15 10l4.553-2.276A1 1 0 0121 8.618v6.764a1 1 0 01-1.447.894L15 14M5 18h8a2 2 0 002-2V8a2 2 0 00-2-2H5a2 2 0 00-2 2v8a2 2 0 002 2z" /></svg>
</div> <p class="text-sm">No episodes found</p>
{{else}} </div>
{{$totalEps := $anime.Episodes}} {{else}}
{{$fallbackSub := .WatchData.FallbackEpisodes.sub}} <div class="flex flex-col gap-1 overflow-y-auto max-h-[70vh] lg:max-h-[calc(100vh-8rem)] pr-2 scrollbar-hide" data-episode-list>
{{if gt $fallbackSub $totalEps}} {{range $episodes}}
{{$totalEps = $fallbackSub}} {{$isCurrent := eq (printf "%v" .MalID) $currentEpID}}
<a href="/anime/{{$anime.MalID}}/watch?ep={{.MalID}}" class="flex items-center gap-3 px-3 py-2 transition-colors hover:bg-white/5 text-left {{if $isCurrent}}bg-accent/20{{end}}" data-episode-id="{{.MalID}}">
<span class="w-10 flex-shrink-0 text-xs font-medium text-neutral-500 tabular-nums">EP{{.MalID}}</span>
<span class="truncate text-sm text-neutral-300" data-episode-title>{{.Title}}</span>
</a>
{{end}}
</div>
{{end}} {{end}}
</div>
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5" data-episode-list>
{{range $episodes}}
{{$isCurrent := eq (printf "%v" .MalID) $currentEpID}}
<a href="/anime/{{$anime.MalID}}/watch?ep={{.MalID}}" class="group flex flex-col overflow-hidden bg-white/5 transition-colors hover:bg-white/10 {{if $isCurrent}}ring-accent ring-2{{end}}" data-episode-id="{{.MalID}}">
<div class="relative aspect-video w-full overflow-hidden bg-black/50">
{{if .Images}}
{{if .Images.Jpg.ImageURL}}
<img src="{{.Images.Jpg.ImageURL}}" alt="{{.Title}}" class="h-full w-full object-cover transition-transform group-hover:scale-105" loading="lazy" />
{{else}}
<div class="flex h-full w-full items-center justify-center text-neutral-600">
<svg class="h-8 w-8" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"/></svg>
</div>
{{end}}
{{else}}
<div class="flex h-full w-full items-center justify-center text-neutral-600">
<svg class="h-8 w-8" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"/></svg>
</div>
{{end}}
<div class="absolute inset-0 flex items-center justify-center bg-black/40 opacity-0 transition-opacity group-hover:opacity-100">
<div class="bg-accent flex h-12 w-12 items-center justify-center rounded-full text-white shadow-lg">
<svg class="ml-1 h-6 w-6" viewBox="0 0 24 24" fill="currentColor"><path d="M8 5v14l11-7z" /></svg>
</div>
</div>
</div>
<div class="flex flex-col gap-1 p-3">
<span class="text-accent text-xs font-semibold">Episode {{.MalID}}</span>
<span class="line-clamp-2 text-sm font-medium text-neutral-200" data-episode-title>{{.Title}}</span>
</div>
</a>
{{end}}
</div>
{{end}}
</div> </div>
{{end}} {{end}}