melosh/mal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplify: remove complex episode fetching, use simple sequential pagination

Browse Source
This commit is contained in:
mkelvers
2026-05-03 14:22:58 +02:00
committed by Mikkel Elvers
parent 4ba4e51d95
commit f2afa6a4e5
3 changed files with 80 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
api/playback/handler.go
View File

@@ -11,8 +11,6 @@ import (
"sort"
"strconv"
"strings"
"sync"
"time"
"mal/integrations/jikan"
database "mal/internal/db"
@@ -58,34 +56,19 @@ func (h *Handler) HandleWatchPage(w http.ResponseWriter, r *http.Request) {
return
}
// Get essential episodes (first and last pages)
allEpisodes, err := h.jikanClient.GetAllEpisodes(r.Context(), id)
if err != nil {
log.Printf("watch error fetching episodes: %v", err)
}
// Fetch any metadata overlays (thumbnails)
videoEpisodes, _ := h.jikanClient.GetVideoEpisodes(r.Context(), id, 1)
videoMeta := make(map[int]jikan.Episode)
for _, ve := range videoEpisodes.Data {
videoMeta[ve.MalID] = ve
}
for i, ep := range allEpisodes {
if ve, ok := videoMeta[ep.MalID]; ok {
if ve.Images != nil && ve.Images.Jpg.ImageURL != "" {
allEpisodes[i].Images = ve.Images
}
// Fetch episodes sequentially (pages are in correct order: 1-100, 101-200, etc.)
pageSize := 100
var allEpisodes []jikan.Episode
for page := 1; ; page++ {
resp, err := h.jikanClient.GetEpisodes(r.Context(), id, page)
if err != nil || len(resp.Data) == 0 {
break
}
}
allEpisodes = append(allEpisodes, resp.Data...)
// Deduplicate and prep the list
seen := make(map[int]bool)
unique := make([]jikan.Episode, 0)
for _, ep := range allEpisodes {
if !seen[ep.MalID] {
seen[ep.MalID] = true
unique = append(unique, ep)
// If we got fewer than pageSize, we've reached the end
if len(resp.Data) < pageSize {
break
}
}
@@ -127,7 +110,7 @@ func (h *Handler) HandleWatchPage(w http.ResponseWriter, r *http.Request) {
log.Printf("watch data error: %v", err)
}
// Update episodes list if fallback has more
// Fill gaps with placeholder episodes if fallback has more
if watchData.FallbackEpisodes != nil {
maxCount := 0
for _, count := range watchData.FallbackEpisodes {
@@ -137,74 +120,34 @@ func (h *Handler) HandleWatchPage(w http.ResponseWriter, r *http.Request) {
}
epMap := make(map[int]jikan.Episode)
for _, ep := range unique {
for _, ep := range allEpisodes {
epMap[ep.MalID] = ep
}
if maxCount > 0 {
var fullList []jikan.Episode
var filled []jikan.Episode
for i := 1; i <= maxCount; i++ {
if ep, ok := epMap[i]; ok {
fullList = append(fullList, ep)
filled = append(filled, ep)
} else {
fullList = append(fullList, jikan.Episode{
filled = append(filled, jikan.Episode{
MalID: i,
Episode: fmt.Sprintf("Episode %d", i),
Title: fmt.Sprintf("Episode %d", i),
})
}
}
unique = fullList
allEpisodes = filled
}
}
// Update episodes list if fallback has more
if watchData.FallbackEpisodes != nil {
maxCount := 0
for _, count := range watchData.FallbackEpisodes {
if count > maxCount {
maxCount = count
}
}
// Ensure we don't have duplicates or missing episodes in the sequence
epMap := make(map[int]jikan.Episode)
for _, ep := range unique {
epMap[ep.MalID] = ep
}
if maxCount > 0 {
var newEpisodes []jikan.Episode
// We build the list from 1 to maxCount to ensure order and completeness
// If we have data from Jikan, we use it. Otherwise we generate a placeholder.
for i := 1; i <= maxCount; i++ {
if ep, ok := epMap[i]; ok {
newEpisodes = append(newEpisodes, ep)
} else {
title := fmt.Sprintf("Episode %d", i)
newEpisodes = append(newEpisodes, jikan.Episode{
MalID: i,
Episode: fmt.Sprintf("Episode %d", i),
Title: title,
Images: &jikan.EpisodeImages{
Jpg: struct {
ImageURL string `json:"image_url"`
}{ImageURL: ""},
},
})
}
}
unique = newEpisodes
}
}
sort.Slice(unique, func(i, j int) bool {
return unique[i].MalID < unique[j].MalID
sort.Slice(allEpisodes, func(i, j int) bool {
return allEpisodes[i].MalID < allEpisodes[j].MalID
})
if err := templates.GetRenderer().ExecuteTemplate(r.Context(), w, "watch.gohtml", map[string]any{
"Anime": anime,
"Episodes": unique,
"Episodes": allEpisodes,
"WatchData": watchData,
"User": user,
"CurrentPath": r.URL.Path,
@@ -418,103 +361,54 @@ func (h *Handler) HandleEpisodeThumbnails(w http.ResponseWriter, r *http.Request
return
}
// Get essential episodes (first and last pages)
allEpisodes, err := h.jikanClient.GetAllEpisodes(r.Context(), id)
if err != nil {
http.Error(w, "failed to get episodes", http.StatusInternalServerError)
return
}
// Also get video episodes for richer metadata (thumbnails) on recent episodes
videoEpisodes, _ := h.jikanClient.GetVideoEpisodes(r.Context(), id, 1)
// Merge metadata
videoMeta := make(map[int]jikan.Episode)
for _, ve := range videoEpisodes.Data {
videoMeta[ve.MalID] = ve
}
for i, ep := range allEpisodes {
if ve, ok := videoMeta[ep.MalID]; ok {
if ve.Images != nil && ve.Images.Jpg.ImageURL != "" {
allEpisodes[i].Images = ve.Images
}
// Fetch episodes sequentially
pageSize := 100
var allEpisodes []jikan.Episode
for page := 1; ; page++ {
resp, err := h.jikanClient.GetEpisodes(r.Context(), id, page)
if err != nil || len(resp.Data) == 0 {
break
}
allEpisodes = append(allEpisodes, resp.Data...)
if len(resp.Data) < pageSize {
break
}
}
// Dedup and sort
seen := make(map[int]bool)
unique := make([]jikan.Episode, 0, len(allEpisodes))
for _, ep := range allEpisodes {
if !seen[ep.MalID] {
seen[ep.MalID] = true
unique = append(unique, ep)
}
}
// Calculate total count from anime info for complete list
// Fill gaps if anime has known total
anime, _ := h.jikanClient.GetAnimeByID(r.Context(), id)
maxCount := anime.Episodes
epMap := make(map[int]jikan.Episode)
for _, ep := range unique {
epMap[ep.MalID] = ep
}
if maxCount > 0 {
var fullList []jikan.Episode
for i := 1; i <= maxCount; i++ {
if anime.Episodes > 0 && anime.Episodes > len(allEpisodes) {
epMap := make(map[int]jikan.Episode)
for _, ep := range allEpisodes {
epMap[ep.MalID] = ep
}
var filled []jikan.Episode
for i := 1; i <= anime.Episodes; i++ {
if ep, ok := epMap[i]; ok {
fullList = append(fullList, ep)
filled = append(filled, ep)
} else {
fullList = append(fullList, jikan.Episode{
filled = append(filled, jikan.Episode{
MalID: i,
Episode: fmt.Sprintf("Episode %d", i),
Title: fmt.Sprintf("Episode %d", i),
})
}
}
unique = fullList
allEpisodes = filled
}
sort.Slice(unique, func(i, j int) bool {
return unique[i].MalID < unique[j].MalID
})
type ThumbResult struct {
type Result struct {
MalID int `json:"mal_id"`
URL string `json:"url"`
Title string `json:"title,omitempty"`
Title string `json:"title"`
}
results := make([]ThumbResult, len(unique))
// Use a semaphore to limit concurrent scraping requests to avoid MAL bans
sem := make(chan struct{}, 2)
var wg sync.WaitGroup
for i := range unique {
wg.Add(1)
go func(idx int) {
defer wg.Done()
sem <- struct{}{} // Acquire
// Add a small jittered delay between requests to avoid 405/429
time.Sleep(time.Duration(200+idx%300) * time.Millisecond)
defer func() { <-sem }() // Release
ep := unique[idx]
imgURL := ep.GetFallbackImage(id)
results[idx] = ThumbResult{
MalID: ep.MalID,
URL: imgURL,
Title: ep.Title,
}
}(i)
results := make([]Result, len(allEpisodes))
for i, ep := range allEpisodes {
results[i] = Result{
MalID: ep.MalID,
Title: ep.Title,
}
}
wg.Wait()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(results)

195
integrations/jikan/episodes.go
View File

@@ -3,118 +3,9 @@ package jikan
import (
"context"
"fmt"
"io"
"log"
"net/http"
"regexp"
"strconv"
"strings"
"time"
)
const bannedImageURL = "https://myanimelist.net/images/icon-banned-youtube.png"
const placeholderImageURL = "https://myanimelist.net/images/episodes/videos/icon-thumbs-not-available.png"
var httpClient = &http.Client{Timeout: 10 * time.Second}
func (e *Episode) GetFallbackImage(animeID int) string {
imageUrl := ""
if e.Images != nil {
imageUrl = e.Images.Jpg.ImageURL
}
// Determining the episode number reliably. Jikan's Episode string can be "Episode 1" or just "1"
episodeNum := 0
if e.Episode != "" {
re := regexp.MustCompile(`\d+`)
match := re.FindString(e.Episode)
if match != "" {
episodeNum, _ = strconv.Atoi(match)
}
}
// For Video episodes, MalID is often the episode number, but let's check
if episodeNum == 0 {
episodeNum = e.MalID
}
// Always trigger scraping if we encounter the banned icon OR the generic placeholder
// OR if there is no image URL at all
if imageUrl == bannedImageURL || imageUrl == placeholderImageURL || imageUrl == "" {
// MAL URLs follow this format: https://myanimelist.net/anime/20/Naruto/episode/1
// The previous format used /_/ which is sometimes rejected with 405
episodeURL := fmt.Sprintf("https://myanimelist.net/anime/%d/slug/episode/%d", animeID, episodeNum)
fallbackURL := scrapeAnimeImageFromEpisodePage(episodeURL, episodeNum)
if fallbackURL != "" {
return fallbackURL
}
}
return imageUrl
}
func scrapeAnimeImageFromEpisodePage(episodeURL string, episodeNum int) string {
req, err := http.NewRequest("GET", episodeURL, nil)
if err != nil {
return ""
}
// Setting User-Agent is important for MAL
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
resp, err := httpClient.Do(req)
if err != nil {
return ""
}
defer resp.Body.Close()
// Log the status code for debugging
if resp.StatusCode != 200 {
log.Printf("[DEBUG] Scraper failed to fetch %s: Status %d", episodeURL, resp.StatusCode)
return ""
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return ""
}
html := string(body)
// MAL sometimes redirects to a URL with a slug.
// We look for the "thumbnail" field in the page source.
// Pattern 1: Look for the specific episode object in the JSON data
episodeStr := strconv.Itoa(episodeNum)
objPattern := regexp.MustCompile(`\{[^{}]*"episode_number":\s*` + episodeStr + `[^{}]*\}`)
match := objPattern.FindString(html)
if match == "" {
// Try a broader search if the strict one fails
objPattern = regexp.MustCompile(`\{[^}]*"episode_number":\s*` + episodeStr + `[^}]*\}`)
match = objPattern.FindString(html)
}
if match != "" {
thumbRe := regexp.MustCompile(`"thumbnail":\s*"([^"]+)"`)
thumbMatch := thumbRe.FindStringSubmatch(match)
if len(thumbMatch) > 1 {
return strings.ReplaceAll(thumbMatch[1], `\/`, `/`)
}
}
// Pattern 2: Fallback to og:image if it's the specific episode page
ogRe := regexp.MustCompile(`<meta\s+property="og:image"\s+content="([^"]+)"`)
ogMatch := ogRe.FindStringSubmatch(html)
if len(ogMatch) > 1 {
// Only use if it looks like an episode thumbnail (contains /episodes/)
if strings.Contains(ogMatch[1], "/episodes/") {
return ogMatch[1]
}
}
return ""
}
func (c *Client) GetEpisodes(ctx context.Context, animeID int, page int) (EpisodesResponse, error) {
if page < 1 {
page = 1
@@ -128,86 +19,14 @@ func (c *Client) GetEpisodes(ctx context.Context, animeID int, page int) (Episod
return result, err
}
func (c *Client) GetVideoEpisodes(ctx context.Context, animeID int, page int) (EpisodesResponse, error) {
if page < 1 {
page = 1
}
cacheKey := fmt.Sprintf("anime:%d:videos:episodes:%d", animeID, page)
var result EpisodesResponse
reqURL := fmt.Sprintf("%s/anime/%d/videos/episodes?page=%d", c.baseURL, animeID, page)
err := c.getWithCache(ctx, cacheKey, 12*time.Hour, reqURL, &result)
return result, err
}
func (c *Client) GetEpisode(ctx context.Context, animeID int, episode int) (EpisodeResponse, error) {
cacheKey := fmt.Sprintf("anime:%d:episode:%d", animeID, episode)
var result EpisodeResponse
reqURL := fmt.Sprintf("%s/anime/%d/episodes/%d", c.baseURL, animeID, episode)
err := c.getWithCache(ctx, cacheKey, 24*time.Hour, reqURL, &result)
return result, err
}
func (c *Client) GetAllEpisodes(ctx context.Context, animeID int) ([]Episode, error) {
// First fetch the anime to get total episodes count
anime, err := c.GetAnimeByID(ctx, animeID)
if err != nil {
return nil, err
}
totalEpisodes := anime.Episodes
if totalEpisodes <= 0 {
resp, err := c.GetEpisodes(ctx, animeID, 1)
func (c *Client) GetEpisodesRange(ctx context.Context, animeID int, startPage, endPage int) ([]Episode, error) {
var all []Episode
for page := startPage; page <= endPage; page++ {
resp, err := c.GetEpisodes(ctx, animeID, page)
if err != nil {
return nil, err
return all, err
}
return resp.Data, nil
all = append(all, resp.Data...)
}
// Jikan /episodes/video (which has thumbnails) returns ~39-40 per page.
// Jikan /episodes (standard) returns 100 per page.
// Since the user wants to prioritize the metadata-rich video clips if possible,
// we will calculate based on the 100-per-page standard endpoint for the full list,
// but the background logic remains the same: last page to first.
pageSize := 100
lastPage := (totalEpisodes + (pageSize - 1)) / pageSize
var allEpisodes []Episode
// Fetch last page first (to get most recent episodes immediately)
lastResp, err := c.GetEpisodes(ctx, animeID, lastPage)
if err == nil {
allEpisodes = append(allEpisodes, lastResp.Data...)
}
// For the rest, fetch them in reverse order in the background
if lastPage > 1 {
go func() {
bgCtx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
// Start from lastPage - 1 and go down to 1
for p := lastPage - 1; p >= 1; p-- {
_, _ = c.GetEpisodes(bgCtx, animeID, p)
// Also pre-fetch the video episodes metadata (39 per page)
// to warm the cache for thumbnails
videoPageSize := 39
vPageStart := ((p-1)*pageSize)/videoPageSize + 1
vPageEnd := (p*pageSize)/videoPageSize + 1
for v := vPageEnd; v >= vPageStart; v-- {
_, _ = c.GetVideoEpisodes(bgCtx, animeID, v)
}
select {
case <-bgCtx.Done():
return
case <-time.After(800 * time.Millisecond):
}
}
}()
}
return allEpisodes, nil
return all, nil
}

70
templates/watch.gohtml
View File

@@ -4,54 +4,30 @@
{{$episodes := .Episodes}}
{{$currentEpID := .CurrentEpID}}
<div class="flex flex-col gap-8 pb-12">
<div id="video-player-container">
{{template "video_player" dict "WatchData" .WatchData "TotalEpisodes" $anime.Episodes}}
<div class="flex flex-col gap-8 pb-12 lg:flex-row lg:gap-6">
<div class="flex-1 min-w-0">
<div id="video-player-container">
{{template "video_player" dict "WatchData" .WatchData "TotalEpisodes" $anime.Episodes}}
</div>
</div>
{{if eq (len $episodes) 0}}
<div class="flex flex-col items-center justify-center gap-2 py-24 text-neutral-400">
<svg class="h-12 w-12 opacity-30" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M15 10l4.553-2.276A1 1 0 0121 8.618v6.764a1 1 0 01-1.447.894L15 14M5 18h8a2 2 0 002-2V8a2 2 0 00-2-2H5a2 2 0 00-2 2v8a2 2 0 002 2z" /></svg>
<p class="text-lg">No episodes found for this anime.</p>
</div>
{{else}}
{{$totalEps := $anime.Episodes}}
{{$fallbackSub := .WatchData.FallbackEpisodes.sub}}
{{if gt $fallbackSub $totalEps}}
{{$totalEps = $fallbackSub}}
<div class="w-full lg:w-80 xl:w-96 flex-shrink-0">
{{if eq (len $episodes) 0}}
<div class="flex flex-col items-center justify-center gap-2 py-12 text-neutral-400">
<svg class="h-10 w-10 opacity-30" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.5"><path stroke-linecap="round" stroke-linejoin="round" d="M15 10l4.553-2.276A1 1 0 0121 8.618v6.764a1 1 0 01-1.447.894L15 14M5 18h8a2 2 0 002-2V8a2 2 0 00-2-2H5a2 2 0 00-2 2v8a2 2 0 002 2z" /></svg>
<p class="text-sm">No episodes found</p>
</div>
{{else}}
<div class="flex flex-col gap-1 overflow-y-auto max-h-[70vh] lg:max-h-[calc(100vh-8rem)] pr-2 scrollbar-hide" data-episode-list>
{{range $episodes}}
{{$isCurrent := eq (printf "%v" .MalID) $currentEpID}}
<a href="/anime/{{$anime.MalID}}/watch?ep={{.MalID}}" class="flex items-center gap-3 px-3 py-2 transition-colors hover:bg-white/5 text-left {{if $isCurrent}}bg-accent/20{{end}}" data-episode-id="{{.MalID}}">
<span class="w-10 flex-shrink-0 text-xs font-medium text-neutral-500 tabular-nums">EP{{.MalID}}</span>
<span class="truncate text-sm text-neutral-300" data-episode-title>{{.Title}}</span>
</a>
{{end}}
</div>
{{end}}
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 xl:grid-cols-5" data-episode-list>
{{range $episodes}}
{{$isCurrent := eq (printf "%v" .MalID) $currentEpID}}
<a href="/anime/{{$anime.MalID}}/watch?ep={{.MalID}}" class="group flex flex-col overflow-hidden bg-white/5 transition-colors hover:bg-white/10 {{if $isCurrent}}ring-accent ring-2{{end}}" data-episode-id="{{.MalID}}">
<div class="relative aspect-video w-full overflow-hidden bg-black/50">
{{if .Images}}
{{if .Images.Jpg.ImageURL}}
<img src="{{.Images.Jpg.ImageURL}}" alt="{{.Title}}" class="h-full w-full object-cover transition-transform group-hover:scale-105" loading="lazy" />
{{else}}
<div class="flex h-full w-full items-center justify-center text-neutral-600">
<svg class="h-8 w-8" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"/></svg>
</div>
{{end}}
{{else}}
<div class="flex h-full w-full items-center justify-center text-neutral-600">
<svg class="h-8 w-8" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M4 16l4.586-4.586a2 2 0 012.828 0L16 16m-2-2l1.586-1.586a2 2 0 012.828 0L20 14m-6-6h.01M6 20h12a2 2 0 002-2V6a2 2 0 00-2-2H6a2 2 0 00-2 2v12a2 2 0 002 2z"/></svg>
</div>
{{end}}
<div class="absolute inset-0 flex items-center justify-center bg-black/40 opacity-0 transition-opacity group-hover:opacity-100">
<div class="bg-accent flex h-12 w-12 items-center justify-center rounded-full text-white shadow-lg">
<svg class="ml-1 h-6 w-6" viewBox="0 0 24 24" fill="currentColor"><path d="M8 5v14l11-7z" /></svg>
</div>
</div>
</div>
<div class="flex flex-col gap-1 p-3">
<span class="text-accent text-xs font-semibold">Episode {{.MalID}}</span>
<span class="line-clamp-2 text-sm font-medium text-neutral-200" data-episode-title>{{.Title}}</span>
</div>
</a>
{{end}}
</div>
{{end}}
</div>
</div>
{{end}}
{{end}}