melosh/mal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: parallelize data fetching and harden rate limiting

Browse Source
This commit is contained in:
mkelvers
2026-05-04 19:53:43 +02:00
parent c97dcae605
commit 326b70036f
4 changed files with 182 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
api/anime/handler.go
View File

@@ -1,18 +1,23 @@
package anime
import (
"context"
"encoding/json"
"errors"
"html"
"log"
"net/http"
"strconv"
"strings"
"time"
"mal/integrations/jikan"
ctxpkg "mal/internal/context"
"mal/internal/db"
"mal/internal/middleware"
"mal/templates"
"golang.org/x/sync/errgroup"
)
type Handler struct {
@@ -60,17 +65,49 @@ func (h *Handler) HandleCatalog(w http.ResponseWriter, r *http.Request) {
return
}
animes, err := h.jikanClient.GetTopAnime(r.Context(), 1)
if err != nil {
log.Printf("top anime error: %v", err)
http.Error(w, "Failed to fetch anime", http.StatusInternalServerError)
return
}
var (
animes jikan.TopAnimeResult
currentlyAiring jikan.TopAnimeResult
cw []database.GetContinueWatchingEntriesRow
watchlist []database.GetUserWatchListRow
)
currentlyAiring, err := h.jikanClient.GetSeasonsNow(r.Context(), 1)
g, gCtx := errgroup.WithContext(r.Context())
g.Go(func() error {
var err error
animes, err = h.jikanClient.GetTopAnime(gCtx, 1)
return err
})
g.Go(func() error {
var err error
currentlyAiring, err = h.jikanClient.GetSeasonsNow(gCtx, 1)
if err != nil {
log.Printf("seasons now error: %v", err)
// non-fatal
return nil // non-fatal
}
return nil
})
user := middleware.GetUser(r.Context())
if user != nil {
g.Go(func() error {
var err error
cw, err = h.db.GetContinueWatchingEntries(gCtx, user.ID)
return err
})
g.Go(func() error {
var err error
watchlist, err = h.db.GetUserWatchList(gCtx, user.ID)
return err
})
}
if err := g.Wait(); err != nil {
log.Printf("catalog fetch error: %v", err)
http.Error(w, "Failed to fetch catalog data", http.StatusInternalServerError)
return
}
if len(animes.Animes) > 6 {
@@ -80,23 +117,12 @@ func (h *Handler) HandleCatalog(w http.ResponseWriter, r *http.Request) {
currentlyAiring.Animes = currentlyAiring.Animes[:6]
}
// Fetch continue watching if logged in (user ID in context, handle this safely)
// We'll skip DB fetch for continue watching for now if it requires complex session parsing
// Actually we should try to fetch it if we can.
var cw []database.GetContinueWatchingEntriesRow
watchlistMap := make(map[int64]bool)
var watchlistIDs []int64
user := middleware.GetUser(r.Context())
userOk := user != nil
if userOk {
cw, _ = h.db.GetContinueWatchingEntries(r.Context(), user.ID)
watchlist, _ := h.db.GetUserWatchList(r.Context(), user.ID)
watchlistIDs = make([]int64, len(watchlist))
watchlistIDs := make([]int64, len(watchlist))
for i, entry := range watchlist {
watchlistMap[entry.AnimeID] = true
watchlistIDs[i] = entry.AnimeID
}
}
if err := templates.GetRenderer().ExecuteTemplate(r.Context(), w, "index.gohtml", map[string]any{
"MostPopular": animes.Animes,
@@ -135,22 +161,28 @@ func (h *Handler) HandleBrowse(w http.ResponseWriter, r *http.Request) {
page = 1
}
res, err := h.jikanClient.SearchAdvanced(r.Context(), q, animeType, status, orderBy, sort, genres, page, 24)
ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
defer cancel()
res, err := h.jikanClient.SearchAdvanced(ctx, q, animeType, status, orderBy, sort, genres, page, 24)
if err != nil {
if errors.Is(err, context.Canceled) {
return
}
log.Printf("browse error: %v", err)
}
if r.Header.Get("HX-Request") == "true" {
watchlistMap := make(map[int]bool)
if user != nil {
watchlist, _ := h.db.GetUserWatchList(r.Context(), user.ID)
watchlist, _ := h.db.GetUserWatchList(ctx, user.ID)
for _, entry := range watchlist {
watchlistMap[int(entry.AnimeID)] = true
}
}
w.Header().Set("Content-Type", "text/html")
err := templates.GetRenderer().ExecuteFragment(r.Context(), w, "browse.gohtml", "anime_card_scroll", map[string]any{
err := templates.GetRenderer().ExecuteFragment(ctx, w, "browse.gohtml", "anime_card_scroll", map[string]any{
"Animes": res.Animes,
"NextPage": page + 1,
"HasNextPage": res.HasNextPage,
@@ -163,20 +195,24 @@ func (h *Handler) HandleBrowse(w http.ResponseWriter, r *http.Request) {
"WatchlistMap": watchlistMap,
})
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Printf("fragment render error: %v", err)
}
}
return
}
genresList, err := h.jikanClient.GetAnimeGenres(r.Context())
genresList, err := h.jikanClient.GetAnimeGenres(ctx)
if err != nil {
if !errors.Is(err, context.Canceled) {
log.Printf("genres error: %v", err)
}
}
watchlistMap := make(map[int]bool)
var watchlistIDs []int64
if user != nil {
watchlist, _ := h.db.GetUserWatchList(r.Context(), user.ID)
watchlist, _ := h.db.GetUserWatchList(ctx, user.ID)
watchlistIDs = make([]int64, len(watchlist))
for i, entry := range watchlist {
watchlistMap[int(entry.AnimeID)] = true
@@ -184,7 +220,7 @@ func (h *Handler) HandleBrowse(w http.ResponseWriter, r *http.Request) {
}
}
if err := templates.GetRenderer().ExecuteTemplate(r.Context(), w, "browse.gohtml", map[string]any{
if err := templates.GetRenderer().ExecuteTemplate(ctx, w, "browse.gohtml", map[string]any{
"User": user,
"CurrentPath": r.URL.Path,
"Query": q,
@@ -200,10 +236,12 @@ func (h *Handler) HandleBrowse(w http.ResponseWriter, r *http.Request) {
"WatchlistMap": watchlistMap,
"WatchlistIDs": watchlistIDs,
}); err != nil {
if !errors.Is(err, context.Canceled) {
log.Printf("render error: %v", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
}
}
func (h *Handler) HandleSearch(w http.ResponseWriter, r *http.Request) {
renderNotFoundPage(r, w)
@@ -329,20 +367,52 @@ func (h *Handler) HandleQuickSearch(w http.ResponseWriter, r *http.Request) {
}
func (h *Handler) HandleDiscover(w http.ResponseWriter, r *http.Request) {
trending, err := h.jikanClient.GetSeasonsNow(r.Context(), 1)
var (
trending jikan.TopAnimeResult
upcoming jikan.TopAnimeResult
top jikan.TopAnimeResult
watchlist []database.GetUserWatchListRow
)
g, gCtx := errgroup.WithContext(r.Context())
g.Go(func() error {
var err error
trending, err = h.jikanClient.GetSeasonsNow(gCtx, 1)
if err != nil {
log.Printf("seasons now error: %v", err)
}
return nil
})
upcoming, err := h.jikanClient.GetSeasonsUpcoming(r.Context(), 1)
g.Go(func() error {
var err error
upcoming, err = h.jikanClient.GetSeasonsUpcoming(gCtx, 1)
if err != nil {
log.Printf("seasons upcoming error: %v", err)
}
return nil
})
top, err := h.jikanClient.GetTopAnime(r.Context(), 1)
g.Go(func() error {
var err error
top, err = h.jikanClient.GetTopAnime(gCtx, 1)
if err != nil {
log.Printf("top anime error: %v", err)
}
return nil
})
user, _ := r.Context().Value(ctxpkg.UserKey).(*database.User)
if user != nil {
g.Go(func() error {
var err error
watchlist, err = h.db.GetUserWatchList(gCtx, user.ID)
return err
})
}
g.Wait()
seen := make(map[int]bool)
uniqueTrending := make([]jikan.Anime, 0)
@@ -378,17 +448,12 @@ func (h *Handler) HandleDiscover(w http.ResponseWriter, r *http.Request) {
}
}
user, _ := r.Context().Value(ctxpkg.UserKey).(*database.User)
watchlistMap := make(map[int64]bool)
var watchlistIDs []int64
if user != nil {
watchlist, _ := h.db.GetUserWatchList(r.Context(), user.ID)
watchlistIDs = make([]int64, len(watchlist))
watchlistIDs := make([]int64, len(watchlist))
for i, entry := range watchlist {
watchlistMap[entry.AnimeID] = true
watchlistIDs[i] = entry.AnimeID
}
}
if err := templates.GetRenderer().ExecuteTemplate(r.Context(), w, "discover.gohtml", map[string]any{
"User": user,

1
go.mod
View File

@@ -16,6 +16,7 @@ require (
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/klauspost/compress v1.17.4 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.43.0 // indirect
golang.org/x/text v0.36.0 // indirect
)

2
go.sum
View File

@@ -47,6 +47,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

75
integrations/jikan/relations.go
View File

@@ -5,10 +5,13 @@ import (
"errors"
"fmt"
"log"
"sort"
"strings"
"time"
"mal/integrations/watchorder"
"golang.org/x/sync/errgroup"
)
const chiakiWatchOrderURL = "https://chiaki.site/?/tools/watch_order/id/%d"
@@ -96,40 +99,74 @@ func (c *Client) GetFullRelations(ctx context.Context, id int) ([]RelationEntry,
return c.currentOnlyRelation(ctx, id)
}
seen := make(map[int]bool)
relations := make([]RelationEntry, 0, len(result.WatchOrder)+1)
type fetchResult struct {
index int
anime Anime
entry watchorder.WatchOrderEntry
}
for _, watchOrderEntry := range result.WatchOrder {
if len(relations) >= maxWatchOrderEntries {
var allowedEntries []watchorder.WatchOrderEntry
seen := make(map[int]bool)
for _, entry := range result.WatchOrder {
if len(allowedEntries) >= maxWatchOrderEntries {
break
}
if !isAllowedWatchOrderType(watchOrderEntry.Type) {
if !isAllowedWatchOrderType(entry.Type) || seen[entry.ID] {
continue
}
if seen[watchOrderEntry.ID] {
continue
seen[entry.ID] = true
allowedEntries = append(allowedEntries, entry)
}
anime, err := c.GetAnimeByID(ctx, watchOrderEntry.ID)
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(3)
results := make(chan fetchResult, len(allowedEntries))
for i, entry := range allowedEntries {
i, entry := i, entry
g.Go(func() error {
anime, err := c.GetAnimeByID(gCtx, entry.ID)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
continue
return nil
}
c.EnqueueAnimeFetchRetry(ctx, watchOrderEntry.ID, err)
log.Printf("relations: skipping related anime %d for root %d: %v", watchOrderEntry.ID, id, err)
continue
c.EnqueueAnimeFetchRetry(gCtx, entry.ID, err)
log.Printf("relations: skipping related anime %d for root %d: %v", entry.ID, id, err)
return nil
}
select {
case results <- fetchResult{index: i, anime: anime, entry: entry}:
case <-gCtx.Done():
}
return nil
})
}
seen[watchOrderEntry.ID] = true
go func() {
g.Wait()
close(results)
}()
fetched := make([]fetchResult, 0, len(allowedEntries))
for res := range results {
fetched = append(fetched, res)
}
// Re-sort because they might have finished out of order
sort.Slice(fetched, func(i, j int) bool {
return fetched[i].index < fetched[j].index
})
relations := make([]RelationEntry, 0, len(fetched)+1)
for _, res := range fetched {
relations = append(relations, RelationEntry{
Anime: anime,
Relation: watchOrderTypeLabel(watchOrderEntry.Type),
IsCurrent: watchOrderEntry.ID == id,
Anime: res.anime,
Relation: watchOrderTypeLabel(res.entry.Type),
IsCurrent: res.entry.ID == id,
IsExtra: false,
})
if watchOrderEntry.ID == id {
if res.entry.ID == id {
relations[len(relations)-1].Relation = "Current"
}
}