refactor: reorganize project structure following go standards
This commit is contained in:
35
integrations/jikan/anime.go
Normal file
35
integrations/jikan/anime.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (c *Client) GetAnimeByID(ctx context.Context, id int) (Anime, error) {
|
||||
cacheKey := fmt.Sprintf("anime:%d", id)
|
||||
|
||||
var cached Anime
|
||||
if c.getCache(ctx, cacheKey, &cached) {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
var result AnimeResponse
|
||||
reqURL := fmt.Sprintf("%s/anime/%d/full", c.baseURL, id)
|
||||
|
||||
if err := c.fetchWithRetry(ctx, reqURL, &result); err != nil {
|
||||
var stale Anime
|
||||
if c.getStaleCache(ctx, cacheKey, &stale) {
|
||||
return stale, nil
|
||||
}
|
||||
return Anime{}, err
|
||||
}
|
||||
|
||||
ttl := time.Hour * 24
|
||||
if result.Data.Status == "Finished Airing" {
|
||||
ttl = time.Hour * 24 * 30
|
||||
}
|
||||
|
||||
c.setCache(ctx, cacheKey, result.Data, ttl)
|
||||
return result.Data, nil
|
||||
}
|
||||
329
integrations/jikan/client.go
Normal file
329
integrations/jikan/client.go
Normal file
@@ -0,0 +1,329 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"mal/internal/db"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
baseURL string
|
||||
db database.Querier
|
||||
retrySignal chan struct{}
|
||||
mu sync.Mutex
|
||||
lastReqTime time.Time
|
||||
}
|
||||
|
||||
func NewClient(db database.Querier) *Client {
|
||||
return &Client{
|
||||
httpClient: &http.Client{Timeout: 10 * time.Second},
|
||||
baseURL: "https://api.jikan.moe/v4",
|
||||
db: db,
|
||||
retrySignal: make(chan struct{}, 1),
|
||||
}
|
||||
}
|
||||
|
||||
type APIError struct {
|
||||
StatusCode int
|
||||
URL string
|
||||
}
|
||||
|
||||
func (e *APIError) Error() string {
|
||||
return fmt.Sprintf("jikan api returned status %d", e.StatusCode)
|
||||
}
|
||||
|
||||
func IsNotFoundError(err error) bool {
|
||||
var apiErr *APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
return apiErr.StatusCode == http.StatusNotFound
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func IsRetryableError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
var apiErr *APIError
|
||||
if errors.As(err, &apiErr) {
|
||||
return isRetryableStatus(apiErr.StatusCode)
|
||||
}
|
||||
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return true
|
||||
}
|
||||
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func isRetryableStatus(statusCode int) bool {
|
||||
if statusCode == http.StatusTooManyRequests {
|
||||
return true
|
||||
}
|
||||
|
||||
return statusCode >= 500 && statusCode <= 504
|
||||
}
|
||||
|
||||
func retryDelay(attempt int) time.Duration {
|
||||
base := 500 * time.Millisecond
|
||||
delay := base * time.Duration(1<<attempt)
|
||||
if delay > 8*time.Second {
|
||||
return 8 * time.Second
|
||||
}
|
||||
|
||||
return delay
|
||||
}
|
||||
|
||||
func parseRetryAfter(value string) (time.Duration, bool) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
seconds, err := strconv.Atoi(trimmed)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
if seconds <= 0 {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return time.Duration(seconds) * time.Second, true
|
||||
}
|
||||
|
||||
func waitForRetry(ctx context.Context, delay time.Duration) error {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("request canceled while retrying jikan request: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func truncateErrorMessage(message string) string {
|
||||
if len(message) <= 400 {
|
||||
return message
|
||||
}
|
||||
|
||||
return message[:400]
|
||||
}
|
||||
|
||||
func (c *Client) notifyRetryWorker() {
|
||||
select {
|
||||
case c.retrySignal <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Client) RetrySignal() <-chan struct{} {
|
||||
return c.retrySignal
|
||||
}
|
||||
|
||||
func (c *Client) EnqueueAnimeFetchRetry(parentCtx context.Context, animeID int, cause error) {
|
||||
if animeID <= 0 || !IsRetryableError(cause) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := c.db.EnqueueAnimeFetchRetry(ctx, database.EnqueueAnimeFetchRetryParams{
|
||||
AnimeID: int64(animeID),
|
||||
LastError: truncateErrorMessage(cause.Error()),
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
c.notifyRetryWorker()
|
||||
}
|
||||
|
||||
func (c *Client) waitRateLimit(ctx context.Context) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
// Jikan has a 3 req/sec limit AND a 60 req/min limit.
|
||||
// 400ms base delay keeps us safely under the 3/sec limit.
|
||||
nextAllowed := c.lastReqTime.Add(400 * time.Millisecond)
|
||||
if now.Before(nextAllowed) {
|
||||
timer := time.NewTimer(nextAllowed.Sub(now))
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-timer.C:
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("request canceled while waiting for rate limit: %w", ctx.Err())
|
||||
}
|
||||
c.lastReqTime = time.Now()
|
||||
} else {
|
||||
c.lastReqTime = now
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) getCache(parentCtx context.Context, key string, out any) bool {
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
data, err := c.db.GetJikanCache(ctx, key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte(data), out)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Client) getStaleCache(parentCtx context.Context, key string, out any) bool {
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
data, err := c.db.GetJikanCacheStale(ctx, key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte(data), out)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (c *Client) setCache(parentCtx context.Context, key string, data any, ttl time.Duration) {
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
bytes, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
_ = c.db.SetJikanCache(ctx, database.SetJikanCacheParams{
|
||||
Key: key,
|
||||
Data: string(bytes),
|
||||
ExpiresAt: time.Now().Add(ttl),
|
||||
})
|
||||
}
|
||||
|
||||
type cacheResult struct {
|
||||
data any
|
||||
hasStale bool
|
||||
}
|
||||
|
||||
func (c *Client) getWithCache(ctx context.Context, cacheKey string, ttl time.Duration, url string, out any) error {
|
||||
if c.getCache(ctx, cacheKey, out) {
|
||||
return nil
|
||||
}
|
||||
|
||||
var stale any
|
||||
hasStale := c.getStaleCache(ctx, cacheKey, &stale)
|
||||
|
||||
if err := c.fetchWithRetry(ctx, url, out); err != nil {
|
||||
if hasStale {
|
||||
staleBytes, marshalErr := json.Marshal(stale)
|
||||
if marshalErr == nil {
|
||||
unmarshalErr := json.Unmarshal(staleBytes, out)
|
||||
if unmarshalErr == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
log.Printf("jikan: stale cache unmarshal failed, falling back to error: %v", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
c.setCache(ctx, cacheKey, out, ttl)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) fetchWithRetry(ctx context.Context, urlStr string, out any) error {
|
||||
maxRetries := 5
|
||||
for attempt := range maxRetries {
|
||||
if err := c.waitRateLimit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, urlStr, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create jikan request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
if attempt < maxRetries-1 && IsRetryableError(err) {
|
||||
if retryErr := waitForRetry(ctx, retryDelay(attempt)); retryErr != nil {
|
||||
return retryErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
return fmt.Errorf("jikan api error: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
apiErr := &APIError{StatusCode: resp.StatusCode, URL: urlStr}
|
||||
retryable := isRetryableStatus(resp.StatusCode)
|
||||
|
||||
retryAfter := time.Duration(0)
|
||||
if parsed, ok := parseRetryAfter(resp.Header.Get("Retry-After")); ok {
|
||||
retryAfter = parsed
|
||||
}
|
||||
|
||||
resp.Body.Close()
|
||||
|
||||
if retryable && attempt < maxRetries-1 {
|
||||
delay := retryDelay(attempt)
|
||||
if retryAfter > delay {
|
||||
delay = retryAfter
|
||||
}
|
||||
|
||||
if retryErr := waitForRetry(ctx, delay); retryErr != nil {
|
||||
return retryErr
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
return apiErr
|
||||
}
|
||||
|
||||
err = json.NewDecoder(resp.Body).Decode(out)
|
||||
resp.Body.Close()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if attempt < maxRetries-1 {
|
||||
if retryErr := waitForRetry(ctx, retryDelay(attempt)); retryErr != nil {
|
||||
return retryErr
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to decode jikan response: %w", err)
|
||||
}
|
||||
|
||||
return fmt.Errorf("max retries exceeded for %s", urlStr)
|
||||
}
|
||||
5
integrations/jikan/constants.go
Normal file
5
integrations/jikan/constants.go
Normal file
@@ -0,0 +1,5 @@
|
||||
package jikan
|
||||
|
||||
import "time"
|
||||
|
||||
const shortCacheTTL = time.Hour
|
||||
20
integrations/jikan/episodes.go
Normal file
20
integrations/jikan/episodes.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (c *Client) GetEpisodes(ctx context.Context, animeID int, page int) (EpisodesResponse, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("anime:%d:episodes:%d", animeID, page)
|
||||
var result EpisodesResponse
|
||||
reqURL := fmt.Sprintf("%s/anime/%d/episodes?page=%d", c.baseURL, animeID, page)
|
||||
|
||||
err := c.getWithCache(ctx, cacheKey, 12*time.Hour, reqURL, &result)
|
||||
return result, err
|
||||
}
|
||||
91
integrations/jikan/recommendations.go
Normal file
91
integrations/jikan/recommendations.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RecommendationEntry struct {
|
||||
Entry struct {
|
||||
MalID int `json:"mal_id"`
|
||||
URL string `json:"url"`
|
||||
Images struct {
|
||||
Webp struct {
|
||||
LargeImageURL string `json:"large_image_url"`
|
||||
} `json:"webp"`
|
||||
} `json:"images"`
|
||||
Title string `json:"title"`
|
||||
} `json:"entry"`
|
||||
Votes int `json:"votes"`
|
||||
}
|
||||
|
||||
type RecommendationsResponse struct {
|
||||
Data []RecommendationEntry `json:"data"`
|
||||
}
|
||||
|
||||
func (c *Client) GetRecommendations(ctx context.Context, animeID int, limit int) ([]Anime, error) {
|
||||
cacheKey := fmt.Sprintf("recs:%d", animeID)
|
||||
|
||||
var cached []Anime
|
||||
if c.getCache(ctx, cacheKey, &cached) {
|
||||
if limit > 0 && len(cached) > limit {
|
||||
return cached[:limit], nil
|
||||
}
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
var result RecommendationsResponse
|
||||
reqURL := fmt.Sprintf("%s/anime/%d/recommendations", c.baseURL, animeID)
|
||||
|
||||
if err := c.fetchWithRetry(ctx, reqURL, &result); err != nil {
|
||||
var stale []Anime
|
||||
if c.getStaleCache(ctx, cacheKey, &stale) {
|
||||
if limit > 0 && len(stale) > limit {
|
||||
return stale[:limit], nil
|
||||
}
|
||||
return stale, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
max := len(result.Data)
|
||||
if limit > 0 && max > limit {
|
||||
max = limit
|
||||
}
|
||||
|
||||
animes := make([]Anime, 0, max)
|
||||
for i := 0; i < max; i++ {
|
||||
rec := result.Data[i]
|
||||
|
||||
var fullAnime Anime
|
||||
animeCacheKey := fmt.Sprintf("anime:%d", rec.Entry.MalID)
|
||||
|
||||
if c.getCache(ctx, animeCacheKey, &fullAnime) {
|
||||
animes = append(animes, fullAnime)
|
||||
} else {
|
||||
anime := Anime{
|
||||
MalID: rec.Entry.MalID,
|
||||
Title: rec.Entry.Title,
|
||||
Images: struct {
|
||||
Jpg struct {
|
||||
LargeImageURL string `json:"large_image_url"`
|
||||
} `json:"jpg"`
|
||||
Webp struct {
|
||||
LargeImageURL string `json:"large_image_url"`
|
||||
} `json:"webp"`
|
||||
}{
|
||||
Webp: struct {
|
||||
LargeImageURL string `json:"large_image_url"`
|
||||
}{
|
||||
LargeImageURL: rec.Entry.Images.Webp.LargeImageURL,
|
||||
},
|
||||
},
|
||||
}
|
||||
animes = append(animes, anime)
|
||||
}
|
||||
}
|
||||
|
||||
c.setCache(ctx, cacheKey, animes, time.Hour*24)
|
||||
return animes, nil
|
||||
}
|
||||
156
integrations/jikan/relations.go
Normal file
156
integrations/jikan/relations.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"mal/integrations/watchorder"
|
||||
)
|
||||
|
||||
const chiakiWatchOrderURL = "https://chiaki.site/?/tools/watch_order/id/%d"
|
||||
const watchOrderCacheTTL = time.Hour * 24
|
||||
const maxWatchOrderEntries = 120
|
||||
|
||||
func watchOrderTypeLabel(value string) string {
|
||||
normalized := strings.ToLower(strings.TrimSpace(value))
|
||||
switch normalized {
|
||||
case "tv":
|
||||
return "TV"
|
||||
case "movie":
|
||||
return "Movie"
|
||||
default:
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
|
||||
func isAllowedWatchOrderType(value string) bool {
|
||||
normalized := strings.ToLower(strings.TrimSpace(value))
|
||||
return normalized == "tv" || normalized == "movie"
|
||||
}
|
||||
|
||||
func relationCacheKey(id int) string {
|
||||
return fmt.Sprintf("relations:watch-order:%d", id)
|
||||
}
|
||||
|
||||
func (c *Client) getWatchOrder(ctx context.Context, id int) (watchorder.WatchOrderResult, error) {
|
||||
cacheKey := relationCacheKey(id)
|
||||
|
||||
var cached watchorder.WatchOrderResult
|
||||
if c.getCache(ctx, cacheKey, &cached) {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
watchOrderURL := fmt.Sprintf(chiakiWatchOrderURL, id)
|
||||
requestCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
result, err := watchorder.FetchWatchOrder(requestCtx, c.httpClient, watchOrderURL)
|
||||
if err != nil {
|
||||
var statusError *watchorder.HTTPStatusError
|
||||
if errors.Is(err, watchorder.ErrWatchOrderMarkupNotFound) {
|
||||
log.Printf("relations: watch-order markup missing for %d (%s): %v", id, watchOrderURL, err)
|
||||
} else if errors.As(err, &statusError) {
|
||||
log.Printf(
|
||||
"relations: watch-order http error for %d (%s): status=%d server=%q cf_ray=%q location=%q content_type=%q body=%q",
|
||||
id,
|
||||
watchOrderURL,
|
||||
statusError.StatusCode,
|
||||
statusError.Server,
|
||||
statusError.CFRay,
|
||||
statusError.Location,
|
||||
statusError.ContentType,
|
||||
statusError.BodyPreview,
|
||||
)
|
||||
} else {
|
||||
log.Printf("relations: watch-order fetch failed for %d (%s): %v", id, watchOrderURL, err)
|
||||
}
|
||||
return watchorder.WatchOrderResult{}, err
|
||||
}
|
||||
|
||||
c.setCache(ctx, cacheKey, result, watchOrderCacheTTL)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) currentOnlyRelation(ctx context.Context, id int) ([]RelationEntry, error) {
|
||||
currentAnime, err := c.GetAnimeByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []RelationEntry{{
|
||||
Anime: currentAnime,
|
||||
Relation: "Current",
|
||||
IsCurrent: true,
|
||||
IsExtra: false,
|
||||
}}, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetFullRelations(ctx context.Context, id int) ([]RelationEntry, error) {
|
||||
result, err := c.getWatchOrder(ctx, id)
|
||||
if err != nil {
|
||||
log.Printf("relations: using current-only fallback for %d: %v", id, err)
|
||||
return c.currentOnlyRelation(ctx, id)
|
||||
}
|
||||
|
||||
seen := make(map[int]bool)
|
||||
relations := make([]RelationEntry, 0, len(result.WatchOrder)+1)
|
||||
|
||||
for _, watchOrderEntry := range result.WatchOrder {
|
||||
if len(relations) >= maxWatchOrderEntries {
|
||||
break
|
||||
}
|
||||
|
||||
if !isAllowedWatchOrderType(watchOrderEntry.Type) {
|
||||
continue
|
||||
}
|
||||
|
||||
if seen[watchOrderEntry.ID] {
|
||||
continue
|
||||
}
|
||||
|
||||
anime, err := c.GetAnimeByID(ctx, watchOrderEntry.ID)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
continue
|
||||
}
|
||||
c.EnqueueAnimeFetchRetry(ctx, watchOrderEntry.ID, err)
|
||||
log.Printf("relations: skipping related anime %d for root %d: %v", watchOrderEntry.ID, id, err)
|
||||
continue
|
||||
}
|
||||
|
||||
seen[watchOrderEntry.ID] = true
|
||||
relations = append(relations, RelationEntry{
|
||||
Anime: anime,
|
||||
Relation: watchOrderTypeLabel(watchOrderEntry.Type),
|
||||
IsCurrent: watchOrderEntry.ID == id,
|
||||
IsExtra: false,
|
||||
})
|
||||
if watchOrderEntry.ID == id {
|
||||
relations[len(relations)-1].Relation = "Current"
|
||||
}
|
||||
}
|
||||
|
||||
if !seen[id] {
|
||||
currentAnime, err := c.GetAnimeByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
relations = append([]RelationEntry{{
|
||||
Anime: currentAnime,
|
||||
Relation: "Current",
|
||||
IsCurrent: true,
|
||||
IsExtra: false,
|
||||
}}, relations...)
|
||||
}
|
||||
|
||||
if len(relations) == 0 {
|
||||
return c.currentOnlyRelation(ctx, id)
|
||||
}
|
||||
|
||||
return relations, nil
|
||||
}
|
||||
69
integrations/jikan/relations_test.go
Normal file
69
integrations/jikan/relations_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package jikan
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsAllowedWatchOrderType(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{name: "tv", input: "tv", want: true},
|
||||
{name: "movie", input: "movie", want: true},
|
||||
{name: "case and whitespace", input: " TV ", want: true},
|
||||
{name: "tv special", input: "tv special", want: false},
|
||||
{name: "ova", input: "ova", want: false},
|
||||
{name: "empty", input: "", want: false},
|
||||
}
|
||||
|
||||
for _, testCase := range tests {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
got := isAllowedWatchOrderType(testCase.input)
|
||||
if got != testCase.want {
|
||||
t.Fatalf("expected %v, got %v", testCase.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchOrderTypeLabel(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{name: "tv", input: "tv", want: "TV"},
|
||||
{name: "movie", input: "movie", want: "Movie"},
|
||||
{name: "trimmed passthrough", input: " tv special ", want: "tv special"},
|
||||
}
|
||||
|
||||
for _, testCase := range tests {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
got := watchOrderTypeLabel(testCase.input)
|
||||
if got != testCase.want {
|
||||
t.Fatalf("expected %q, got %q", testCase.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowedWatchOrderTypeFromDataset(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want bool
|
||||
}{
|
||||
{name: "label tv", input: "TV", want: true},
|
||||
{name: "label movie", input: "Movie", want: true},
|
||||
{name: "label special", input: "Special", want: false},
|
||||
}
|
||||
|
||||
for _, testCase := range tests {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
got := isAllowedWatchOrderType(testCase.input)
|
||||
if got != testCase.want {
|
||||
t.Fatalf("expected %v, got %v", testCase.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
84
integrations/jikan/search.go
Normal file
84
integrations/jikan/search.go
Normal file
@@ -0,0 +1,84 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
func (c *Client) Search(ctx context.Context, query string, page int) (SearchResult, error) {
|
||||
return c.search(ctx, query, page, 0)
|
||||
}
|
||||
|
||||
func (c *Client) SearchWithLimit(ctx context.Context, query string, page int, limit int) (SearchResult, error) {
|
||||
return c.search(ctx, query, page, limit)
|
||||
}
|
||||
|
||||
func (c *Client) search(ctx context.Context, query string, page int, limit int) (SearchResult, error) {
|
||||
if query == "" {
|
||||
return SearchResult{}, nil
|
||||
}
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
if limit < 0 {
|
||||
limit = 0
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("search:%s:%d:%d", query, page, limit)
|
||||
|
||||
var result SearchResponse
|
||||
reqURL := fmt.Sprintf("%s/anime?q=%s&page=%d", c.baseURL, url.QueryEscape(query), page)
|
||||
if limit > 0 {
|
||||
reqURL = fmt.Sprintf("%s&limit=%d", reqURL, limit)
|
||||
}
|
||||
|
||||
if err := c.getWithCache(ctx, cacheKey, shortCacheTTL, reqURL, &result); err != nil {
|
||||
if limit > 0 && IsRetryableError(err) {
|
||||
fallbackURL := fmt.Sprintf("%s/anime?q=%s&page=%d", c.baseURL, url.QueryEscape(query), page)
|
||||
if fallbackErr := c.fetchWithRetry(ctx, fallbackURL, &result); fallbackErr == nil {
|
||||
res := SearchResult{
|
||||
Animes: result.Data,
|
||||
HasNextPage: result.Pagination.HasNextPage,
|
||||
}
|
||||
c.setCache(ctx, cacheKey, res, shortCacheTTL)
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
|
||||
var stale SearchResult
|
||||
if c.getStaleCache(ctx, cacheKey, &stale) {
|
||||
return stale, nil
|
||||
}
|
||||
|
||||
return SearchResult{}, err
|
||||
}
|
||||
|
||||
return SearchResult{
|
||||
Animes: result.Data,
|
||||
HasNextPage: result.Pagination.HasNextPage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetTopAnime(ctx context.Context, page int) (TopAnimeResult, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
cacheKey := fmt.Sprintf("top:%d", page)
|
||||
|
||||
var result TopAnimeResponse
|
||||
reqURL := fmt.Sprintf("%s/top/anime?page=%d", c.baseURL, page)
|
||||
|
||||
if err := c.getWithCache(ctx, cacheKey, shortCacheTTL, reqURL, &result); err != nil {
|
||||
var stale TopAnimeResult
|
||||
if c.getStaleCache(ctx, cacheKey, &stale) {
|
||||
return stale, nil
|
||||
}
|
||||
return TopAnimeResult{}, err
|
||||
}
|
||||
|
||||
return TopAnimeResult{
|
||||
Animes: result.Data,
|
||||
HasNextPage: result.Pagination.HasNextPage,
|
||||
}, nil
|
||||
}
|
||||
85
integrations/jikan/seasons.go
Normal file
85
integrations/jikan/seasons.go
Normal file
@@ -0,0 +1,85 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ScheduleResult struct {
|
||||
Animes []Anime
|
||||
HasNextPage bool
|
||||
}
|
||||
|
||||
func (c *Client) GetSchedule(ctx context.Context, day string) (ScheduleResult, error) {
|
||||
day = strings.ToLower(day)
|
||||
cacheKey := fmt.Sprintf("schedule_%s", day)
|
||||
|
||||
var result TopAnimeResponse
|
||||
reqURL := fmt.Sprintf("%s/schedules?filter=%s&sfw=true", c.baseURL, day)
|
||||
|
||||
err := c.getWithCache(ctx, cacheKey, shortCacheTTL, reqURL, &result)
|
||||
if err != nil {
|
||||
return ScheduleResult{}, err
|
||||
}
|
||||
|
||||
return ScheduleResult{
|
||||
Animes: result.Data,
|
||||
HasNextPage: result.Pagination.HasNextPage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetFullSchedule(ctx context.Context) (map[string][]Anime, error) {
|
||||
days := []string{"monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"}
|
||||
schedule := make(map[string][]Anime)
|
||||
|
||||
for _, day := range days {
|
||||
res, err := c.GetSchedule(ctx, day)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch %s schedule: %w", day, err)
|
||||
}
|
||||
schedule[day] = res.Animes
|
||||
}
|
||||
|
||||
return schedule, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetSeasonsNow(ctx context.Context, page int) (TopAnimeResult, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
cacheKey := fmt.Sprintf("seasons_now:%d", page)
|
||||
|
||||
var result TopAnimeResponse
|
||||
reqURL := fmt.Sprintf("%s/seasons/now?page=%d", c.baseURL, page)
|
||||
|
||||
err := c.getWithCache(ctx, cacheKey, shortCacheTTL, reqURL, &result)
|
||||
if err != nil {
|
||||
return TopAnimeResult{}, err
|
||||
}
|
||||
|
||||
return TopAnimeResult{
|
||||
Animes: result.Data,
|
||||
HasNextPage: result.Pagination.HasNextPage,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetSeasonsUpcoming(ctx context.Context, page int) (TopAnimeResult, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
cacheKey := fmt.Sprintf("seasons_upcoming:%d", page)
|
||||
|
||||
var result TopAnimeResponse
|
||||
reqURL := fmt.Sprintf("%s/seasons/upcoming?page=%d", c.baseURL, page)
|
||||
|
||||
err := c.getWithCache(ctx, cacheKey, shortCacheTTL, reqURL, &result)
|
||||
if err != nil {
|
||||
return TopAnimeResult{}, err
|
||||
}
|
||||
|
||||
return TopAnimeResult{
|
||||
Animes: result.Data,
|
||||
HasNextPage: result.Pagination.HasNextPage,
|
||||
}, nil
|
||||
}
|
||||
86
integrations/jikan/studio.go
Normal file
86
integrations/jikan/studio.go
Normal file
@@ -0,0 +1,86 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
type ProducerResponse struct {
|
||||
Data struct {
|
||||
MalID int `json:"mal_id"`
|
||||
Titles []struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
} `json:"titles"`
|
||||
Images struct {
|
||||
Jpg struct {
|
||||
ImageURL string `json:"image_url"`
|
||||
} `json:"jpg"`
|
||||
} `json:"images"`
|
||||
Favorites int `json:"favorites"`
|
||||
Established string `json:"established"`
|
||||
About string `json:"about"`
|
||||
Count int `json:"count"`
|
||||
External []struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
} `json:"external"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (c *Client) GetAnimeByProducer(ctx context.Context, producerID int, page int) (StudioAnimeResult, error) {
|
||||
if page < 1 {
|
||||
page = 1
|
||||
}
|
||||
|
||||
cacheKey := fmt.Sprintf("producer:%d:%d", producerID, page)
|
||||
|
||||
var cached StudioAnimeResult
|
||||
if c.getCache(ctx, cacheKey, &cached) {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
var stale StudioAnimeResult
|
||||
hasStale := c.getStaleCache(ctx, cacheKey, &stale)
|
||||
|
||||
var result SearchResponse
|
||||
reqURL := fmt.Sprintf("%s/anime?producers=%d&page=%d", c.baseURL, producerID, page)
|
||||
|
||||
if err := c.fetchWithRetry(ctx, reqURL, &result); err != nil {
|
||||
if hasStale {
|
||||
return stale, nil
|
||||
}
|
||||
return StudioAnimeResult{}, err
|
||||
}
|
||||
|
||||
producerName := ""
|
||||
var producerRes ProducerResponse
|
||||
producerURL := fmt.Sprintf("%s/producers/%d", c.baseURL, producerID)
|
||||
if err := c.fetchWithRetry(ctx, producerURL, &producerRes); err == nil {
|
||||
for _, title := range producerRes.Data.Titles {
|
||||
if title.Type == "Default" {
|
||||
producerName = title.Title
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res := StudioAnimeResult{
|
||||
Animes: result.Data,
|
||||
HasNextPage: result.Pagination.HasNextPage,
|
||||
StudioName: producerName,
|
||||
}
|
||||
|
||||
c.setCache(ctx, cacheKey, res, shortCacheTTL)
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *Client) GetProducerByID(ctx context.Context, producerID int) (ProducerResponse, error) {
|
||||
cacheKey := fmt.Sprintf("producer:info:%d", producerID)
|
||||
|
||||
var result ProducerResponse
|
||||
reqURL := fmt.Sprintf("%s/producers/%d/full", c.baseURL, producerID)
|
||||
|
||||
err := c.getWithCache(ctx, cacheKey, shortCacheTTL, reqURL, &result)
|
||||
return result, err
|
||||
}
|
||||
96
integrations/jikan/studio_test.go
Normal file
96
integrations/jikan/studio_test.go
Normal file
@@ -0,0 +1,96 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"mal/internal/db"
|
||||
)
|
||||
|
||||
type staleCacheQuerier struct {
|
||||
database.Querier
|
||||
staleJSON string
|
||||
}
|
||||
|
||||
func (q *staleCacheQuerier) GetJikanCache(ctx context.Context, key string) (string, error) {
|
||||
return "", sql.ErrNoRows
|
||||
}
|
||||
|
||||
func (q *staleCacheQuerier) GetJikanCacheStale(ctx context.Context, key string) (string, error) {
|
||||
if q.staleJSON == "" {
|
||||
return "", sql.ErrNoRows
|
||||
}
|
||||
|
||||
return q.staleJSON, nil
|
||||
}
|
||||
|
||||
func TestGetProducerByID_UsesStaleCacheOnFetchFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
q := &staleCacheQuerier{
|
||||
staleJSON: `{"data":{"mal_id":7,"about":"stale about"}}`,
|
||||
}
|
||||
|
||||
client := NewClient(q)
|
||||
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer testServer.Close()
|
||||
|
||||
client.baseURL = testServer.URL
|
||||
client.httpClient = testServer.Client()
|
||||
|
||||
result, err := client.GetProducerByID(context.Background(), 7)
|
||||
if err != nil {
|
||||
t.Fatalf("expected stale cache result, got error: %v", err)
|
||||
}
|
||||
|
||||
if result.Data.MalID != 7 {
|
||||
t.Fatalf("expected stale mal_id 7, got %d", result.Data.MalID)
|
||||
}
|
||||
|
||||
if result.Data.About != "stale about" {
|
||||
t.Fatalf("expected stale about field, got %q", result.Data.About)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAnimeByProducer_UsesStaleCacheOnFetchFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
q := &staleCacheQuerier{
|
||||
staleJSON: `{"Animes":[{"mal_id":42,"title":"Stale Anime"}],"HasNextPage":true,"StudioName":"Stale Studio"}`,
|
||||
}
|
||||
|
||||
client := NewClient(q)
|
||||
testServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer testServer.Close()
|
||||
|
||||
client.baseURL = testServer.URL
|
||||
client.httpClient = testServer.Client()
|
||||
|
||||
result, err := client.GetAnimeByProducer(context.Background(), 9, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("expected stale cache result, got error: %v", err)
|
||||
}
|
||||
|
||||
if len(result.Animes) != 1 {
|
||||
t.Fatalf("expected one stale anime, got %d", len(result.Animes))
|
||||
}
|
||||
|
||||
if result.Animes[0].MalID != 42 {
|
||||
t.Fatalf("expected stale anime mal_id 42, got %d", result.Animes[0].MalID)
|
||||
}
|
||||
|
||||
if !result.HasNextPage {
|
||||
t.Fatal("expected stale has_next_page=true")
|
||||
}
|
||||
|
||||
if result.StudioName != "Stale Studio" {
|
||||
t.Fatalf("expected stale studio name, got %q", result.StudioName)
|
||||
}
|
||||
}
|
||||
202
integrations/jikan/types.go
Normal file
202
integrations/jikan/types.go
Normal file
@@ -0,0 +1,202 @@
|
||||
package jikan
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type SearchResult struct {
|
||||
Animes []Anime
|
||||
HasNextPage bool
|
||||
}
|
||||
|
||||
type TopAnimeResult struct {
|
||||
Animes []Anime
|
||||
HasNextPage bool
|
||||
}
|
||||
|
||||
type StudioAnimeResult struct {
|
||||
Animes []Anime
|
||||
HasNextPage bool
|
||||
StudioName string
|
||||
}
|
||||
|
||||
type NamedEntity struct {
|
||||
MalID int `json:"mal_id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type Aired struct {
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
String string `json:"string"`
|
||||
}
|
||||
|
||||
type Anime struct {
|
||||
MalID int `json:"mal_id"`
|
||||
Title string `json:"title"`
|
||||
TitleEnglish string `json:"title_english"`
|
||||
TitleJapanese string `json:"title_japanese"`
|
||||
TitleSynonyms []string `json:"title_synonyms"`
|
||||
Images struct {
|
||||
Jpg struct {
|
||||
LargeImageURL string `json:"large_image_url"`
|
||||
} `json:"jpg"`
|
||||
Webp struct {
|
||||
LargeImageURL string `json:"large_image_url"`
|
||||
} `json:"webp"`
|
||||
} `json:"images"`
|
||||
Synopsis string `json:"synopsis"`
|
||||
Rank int `json:"rank"`
|
||||
Popularity int `json:"popularity"`
|
||||
Status string `json:"status"`
|
||||
Airing bool `json:"airing"`
|
||||
Episodes int `json:"episodes"`
|
||||
Season string `json:"season"`
|
||||
Year int `json:"year"`
|
||||
Type string `json:"type"`
|
||||
Rating string `json:"rating"`
|
||||
Duration string `json:"duration"`
|
||||
Aired Aired `json:"aired"`
|
||||
Genres []NamedEntity `json:"genres"`
|
||||
Studios []NamedEntity `json:"studios"`
|
||||
Producers []NamedEntity `json:"producers"`
|
||||
Themes []NamedEntity `json:"themes"`
|
||||
Source string `json:"source"`
|
||||
Demographics []NamedEntity `json:"demographics"`
|
||||
Broadcast struct {
|
||||
Day string `json:"day"`
|
||||
Time string `json:"time"`
|
||||
Timezone string `json:"timezone"`
|
||||
String string `json:"string"`
|
||||
} `json:"broadcast"`
|
||||
Streaming []struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
} `json:"streaming"`
|
||||
Relations []JikanRelationGroup `json:"relations"`
|
||||
}
|
||||
|
||||
func (a Anime) ImageURL() string {
|
||||
return a.Images.Webp.LargeImageURL
|
||||
}
|
||||
|
||||
func (a Anime) ShortRating() string {
|
||||
if a.Rating == "" {
|
||||
return ""
|
||||
}
|
||||
// Rating format: "PG-13 - Teens 13 or older"
|
||||
for i, c := range a.Rating {
|
||||
if c == ' ' && i > 0 {
|
||||
return a.Rating[:i]
|
||||
}
|
||||
}
|
||||
return a.Rating
|
||||
}
|
||||
|
||||
func (a Anime) ShortDuration() string {
|
||||
if a.Duration == "" {
|
||||
return ""
|
||||
}
|
||||
// Duration format: "23 min per ep" or "1 hr 30 min"
|
||||
var num string
|
||||
for _, c := range a.Duration {
|
||||
if c >= '0' && c <= '9' {
|
||||
num += string(c)
|
||||
} else if c == ' ' && num != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
if num != "" {
|
||||
return num + "m"
|
||||
}
|
||||
return a.Duration
|
||||
}
|
||||
|
||||
func (a Anime) Premiered() string {
|
||||
if a.Season != "" && a.Year > 0 {
|
||||
return fmt.Sprintf("%s %d", seasonLabel(a.Season), a.Year)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func seasonLabel(season string) string {
|
||||
switch strings.ToLower(season) {
|
||||
case "winter":
|
||||
return "Winter"
|
||||
case "spring":
|
||||
return "Spring"
|
||||
case "summer":
|
||||
return "Summer"
|
||||
case "fall", "autumn":
|
||||
return "Fall"
|
||||
default:
|
||||
if season == "" {
|
||||
return ""
|
||||
}
|
||||
return strings.ToUpper(season[:1]) + strings.ToLower(season[1:])
|
||||
}
|
||||
}
|
||||
|
||||
type AnimeResponse struct {
|
||||
Data Anime `json:"data"`
|
||||
}
|
||||
|
||||
type SearchResponse struct {
|
||||
Data []Anime `json:"data"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
type Pagination struct {
|
||||
HasNextPage bool `json:"has_next_page"`
|
||||
}
|
||||
|
||||
type TopAnimeResponse struct {
|
||||
Data []Anime `json:"data"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
type Episode struct {
|
||||
MalID int `json:"mal_id"`
|
||||
Title string `json:"title"`
|
||||
Filler bool `json:"filler"`
|
||||
Recap bool `json:"recap"`
|
||||
}
|
||||
|
||||
type EpisodesResponse struct {
|
||||
Data []Episode `json:"data"`
|
||||
Pagination Pagination `json:"pagination"`
|
||||
}
|
||||
|
||||
type JikanRelationEntry struct {
|
||||
MalID int `json:"mal_id"`
|
||||
Type string `json:"type"`
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type JikanRelationGroup struct {
|
||||
Relation string `json:"relation"`
|
||||
Entry []JikanRelationEntry `json:"entry"`
|
||||
}
|
||||
|
||||
type JikanRelationsResponse struct {
|
||||
Data []JikanRelationGroup `json:"data"`
|
||||
}
|
||||
|
||||
type RelationEntry struct {
|
||||
Anime Anime
|
||||
Relation string
|
||||
IsCurrent bool
|
||||
IsExtra bool
|
||||
}
|
||||
|
||||
func (a Anime) DisplayTitle() string {
|
||||
if a.TitleEnglish != "" {
|
||||
return a.TitleEnglish
|
||||
}
|
||||
if a.TitleJapanese != "" {
|
||||
return a.TitleJapanese
|
||||
}
|
||||
return a.Title
|
||||
}
|
||||
397
integrations/watchorder/watch_order.go
Normal file
397
integrations/watchorder/watch_order.go
Normal file
@@ -0,0 +1,397 @@
|
||||
package watchorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
|
||||
|
||||
var idPattern = regexp.MustCompile(`/id/(\d+)`)
|
||||
var malLinkPattern = regexp.MustCompile(`myanimelist\.net/anime/(\d+)`)
|
||||
|
||||
var ErrInvalidWatchOrderURL = errors.New("invalid watch order url")
|
||||
var ErrWatchOrderMarkupNotFound = errors.New("watch order markup not found")
|
||||
|
||||
type HTTPStatusError struct {
|
||||
StatusCode int
|
||||
URL string
|
||||
Server string
|
||||
CFRay string
|
||||
Location string
|
||||
ContentType string
|
||||
BodyPreview string
|
||||
}
|
||||
|
||||
func (e *HTTPStatusError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"unexpected status code: %d (url=%s server=%s cf_ray=%s location=%s content_type=%s body=%q)",
|
||||
e.StatusCode,
|
||||
e.URL,
|
||||
e.Server,
|
||||
e.CFRay,
|
||||
e.Location,
|
||||
e.ContentType,
|
||||
e.BodyPreview,
|
||||
)
|
||||
}
|
||||
|
||||
type WatchOrderEntry struct {
|
||||
ID int `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
TitleAlt string `json:"title_alt,omitempty"`
|
||||
}
|
||||
|
||||
type WatchOrderResult struct {
|
||||
ID int `json:"id"`
|
||||
WatchOrder []WatchOrderEntry `json:"watch_order"`
|
||||
}
|
||||
|
||||
type watchOrderRow struct {
|
||||
id int
|
||||
typeID int
|
||||
title string
|
||||
alternativeTitle string
|
||||
}
|
||||
|
||||
func parseRootID(url string) (int, error) {
|
||||
match := idPattern.FindStringSubmatch(url)
|
||||
if len(match) != 2 {
|
||||
return 0, ErrInvalidWatchOrderURL
|
||||
}
|
||||
|
||||
id, err := strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
return 0, ErrInvalidWatchOrderURL
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func addCommonHeaders(request *http.Request) {
|
||||
request.Header.Set("User-Agent", defaultUserAgent)
|
||||
request.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
|
||||
request.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
request.Header.Set("Referer", "https://chiaki.site/")
|
||||
request.Header.Set("Cache-Control", "no-cache")
|
||||
}
|
||||
|
||||
func fetchDocument(ctx context.Context, httpClient *http.Client, url string) (*goquery.Document, error) {
|
||||
client := httpClient
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
addCommonHeaders(request)
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 512))
|
||||
return nil, &HTTPStatusError{
|
||||
StatusCode: response.StatusCode,
|
||||
URL: url,
|
||||
Server: strings.TrimSpace(response.Header.Get("Server")),
|
||||
CFRay: strings.TrimSpace(response.Header.Get("CF-Ray")),
|
||||
Location: strings.TrimSpace(response.Header.Get("Location")),
|
||||
ContentType: strings.TrimSpace(response.Header.Get("Content-Type")),
|
||||
BodyPreview: strings.Join(strings.Fields(strings.TrimSpace(string(body))), " "),
|
||||
}
|
||||
}
|
||||
|
||||
document, err := goquery.NewDocumentFromReader(response.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse html: %w", err)
|
||||
}
|
||||
|
||||
return document, nil
|
||||
}
|
||||
|
||||
func extractTypeLabelsByID(doc *goquery.Document) map[int]string {
|
||||
typeLabels := make(map[int]string)
|
||||
|
||||
doc.Find("#wo_type_filter label").Each(func(_ int, selection *goquery.Selection) {
|
||||
input := selection.Find("input[type='checkbox']")
|
||||
rawID, exists := input.Attr("value")
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
typeID, err := strconv.Atoi(strings.TrimSpace(rawID))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
label := strings.TrimSpace(selection.Text())
|
||||
if label == "" {
|
||||
return
|
||||
}
|
||||
|
||||
typeLabels[typeID] = label
|
||||
})
|
||||
|
||||
return typeLabels
|
||||
}
|
||||
|
||||
func parseAttrInt(selection *goquery.Selection, attrName string) (int, bool) {
|
||||
rawValue, exists := selection.Attr(attrName)
|
||||
if !exists {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
value, err := strconv.Atoi(strings.TrimSpace(rawValue))
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return value, true
|
||||
}
|
||||
|
||||
func extractRows(doc *goquery.Document) []watchOrderRow {
|
||||
rows := make([]watchOrderRow, 0)
|
||||
|
||||
doc.Find("tr[data-id]").Each(func(_ int, selection *goquery.Selection) {
|
||||
id, ok := parseAttrInt(selection, "data-id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
typeID, ok := parseAttrInt(selection, "data-type")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(selection.Find(".wo_title").First().Text())
|
||||
alternativeTitle := strings.TrimSpace(selection.Find(".uk-text-small").First().Text())
|
||||
|
||||
rows = append(rows, watchOrderRow{
|
||||
id: id,
|
||||
typeID: typeID,
|
||||
title: title,
|
||||
alternativeTitle: alternativeTitle,
|
||||
})
|
||||
})
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
func hasWatchOrderTable(doc *goquery.Document) bool {
|
||||
return doc.Find("#wo_list").Length() > 0
|
||||
}
|
||||
|
||||
func shouldTryProxy(err error) bool {
|
||||
var statusError *HTTPStatusError
|
||||
if errors.As(err, &statusError) {
|
||||
return statusError.StatusCode == http.StatusForbidden || statusError.StatusCode == http.StatusTooManyRequests || statusError.StatusCode == http.StatusServiceUnavailable
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func toJinaProxyURL(url string) string {
|
||||
trimmed := strings.TrimPrefix(strings.TrimPrefix(url, "https://"), "http://")
|
||||
return "https://r.jina.ai/http://" + trimmed
|
||||
}
|
||||
|
||||
func fetchProxyText(ctx context.Context, httpClient *http.Client, url string) (string, error) {
|
||||
client := httpClient
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, toJinaProxyURL(url), nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create proxy request: %w", err)
|
||||
}
|
||||
|
||||
addCommonHeaders(request)
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("proxy request failed: %w", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("proxy status %d", response.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 2*1024*1024))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read proxy response: %w", err)
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func parseJinaEntries(text string) []WatchOrderEntry {
|
||||
lines := strings.Split(text, "\n")
|
||||
entries := make([]WatchOrderEntry, 0)
|
||||
seen := make(map[int]bool)
|
||||
|
||||
for index, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if !strings.Contains(trimmed, "myanimelist.net/anime/") || !strings.Contains(trimmed, "|") {
|
||||
continue
|
||||
}
|
||||
|
||||
idMatch := malLinkPattern.FindStringSubmatch(trimmed)
|
||||
if len(idMatch) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
id, err := strconv.Atoi(idMatch[1])
|
||||
if err != nil || seen[id] {
|
||||
continue
|
||||
}
|
||||
|
||||
parts := strings.Split(trimmed, "|")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
typeName := strings.TrimSpace(parts[1])
|
||||
if typeName == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
title, titleAlt := titleFromContext(lines, index)
|
||||
entries = append(entries, WatchOrderEntry{
|
||||
ID: id,
|
||||
Type: typeName,
|
||||
Title: title,
|
||||
TitleAlt: titleAlt,
|
||||
})
|
||||
seen[id] = true
|
||||
}
|
||||
|
||||
return entries
|
||||
}
|
||||
|
||||
func isNoiseTitleLine(value string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(value))
|
||||
if lower == "" {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(lower, "title:") || strings.HasPrefix(lower, "url source:") || strings.HasPrefix(lower, "markdown content:") {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.Contains(lower, "/ watch order") {
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasPrefix(lower, "http://") || strings.HasPrefix(lower, "https://") {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func titleFromContext(lines []string, metaIndex int) (string, string) {
|
||||
collected := make([]string, 0, 2)
|
||||
|
||||
for idx := metaIndex - 1; idx >= 0 && len(collected) < 2; idx-- {
|
||||
candidate := strings.TrimSpace(lines[idx])
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if isNoiseTitleLine(candidate) {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(candidate, "myanimelist.net/anime/") {
|
||||
continue
|
||||
}
|
||||
|
||||
collected = append(collected, candidate)
|
||||
}
|
||||
|
||||
if len(collected) == 0 {
|
||||
return "", ""
|
||||
}
|
||||
|
||||
if len(collected) == 1 {
|
||||
return collected[0], ""
|
||||
}
|
||||
|
||||
return collected[1], collected[0]
|
||||
}
|
||||
|
||||
func fetchViaProxy(ctx context.Context, httpClient *http.Client, url string, rootID int) (WatchOrderResult, error) {
|
||||
proxyText, err := fetchProxyText(ctx, httpClient, url)
|
||||
if err != nil {
|
||||
return WatchOrderResult{}, err
|
||||
}
|
||||
|
||||
entries := parseJinaEntries(proxyText)
|
||||
if len(entries) == 0 {
|
||||
return WatchOrderResult{}, ErrWatchOrderMarkupNotFound
|
||||
}
|
||||
|
||||
return WatchOrderResult{ID: rootID, WatchOrder: entries}, nil
|
||||
}
|
||||
|
||||
func FetchWatchOrder(ctx context.Context, httpClient *http.Client, url string) (WatchOrderResult, error) {
|
||||
rootID, err := parseRootID(url)
|
||||
if err != nil {
|
||||
return WatchOrderResult{}, err
|
||||
}
|
||||
|
||||
doc, err := fetchDocument(ctx, httpClient, url)
|
||||
if err != nil {
|
||||
if shouldTryProxy(err) {
|
||||
return fetchViaProxy(ctx, httpClient, url, rootID)
|
||||
}
|
||||
return WatchOrderResult{}, err
|
||||
}
|
||||
|
||||
if !hasWatchOrderTable(doc) {
|
||||
return fetchViaProxy(ctx, httpClient, url, rootID)
|
||||
}
|
||||
|
||||
rows := extractRows(doc)
|
||||
if len(rows) == 0 {
|
||||
return WatchOrderResult{ID: rootID, WatchOrder: []WatchOrderEntry{}}, nil
|
||||
}
|
||||
|
||||
typeByID := extractTypeLabelsByID(doc)
|
||||
|
||||
entries := make([]WatchOrderEntry, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
typeName := strings.TrimSpace(typeByID[row.typeID])
|
||||
|
||||
entries = append(entries, WatchOrderEntry{
|
||||
ID: row.id,
|
||||
Type: typeName,
|
||||
Title: row.title,
|
||||
TitleAlt: row.alternativeTitle,
|
||||
})
|
||||
}
|
||||
|
||||
return WatchOrderResult{ID: rootID, WatchOrder: entries}, nil
|
||||
}
|
||||
212
integrations/watchorder/watch_order_test.go
Normal file
212
integrations/watchorder/watch_order_test.go
Normal file
@@ -0,0 +1,212 @@
|
||||
package watchorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testServer(body string) *httptest.Server {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(body))
|
||||
})
|
||||
|
||||
return httptest.NewServer(handler)
|
||||
}
|
||||
|
||||
func testHTMLWithMetadata() string {
|
||||
return `
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<div id="wo_type_filter">
|
||||
<label><input type="checkbox" value="1" checked> TV</label>
|
||||
<label><input type="checkbox" value="3" checked> Movie</label>
|
||||
</div>
|
||||
<table id="wo_list">
|
||||
<tr data-id="442" data-anilist-id="442" data-type="3">
|
||||
<td>
|
||||
<span class="wo_title">Naruto Movie 1</span>
|
||||
<span class="uk-text-small">Naruto the Movie 1</span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
|
||||
func testHTMLEmptyRows() string {
|
||||
return `
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<div id="wo_type_filter">
|
||||
<label><input type="checkbox" value="1" checked> TV</label>
|
||||
<label><input type="checkbox" value="3" checked> Movie</label>
|
||||
</div>
|
||||
<table id="wo_list"></table>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
|
||||
func TestFetchWatchOrder_OutputShape(t *testing.T) {
|
||||
server := testServer(testHTMLWithMetadata())
|
||||
defer server.Close()
|
||||
|
||||
url := server.URL + "/?/tools/watch_order/id/442"
|
||||
result, err := FetchWatchOrder(context.Background(), &http.Client{Timeout: time.Second}, url)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if result.ID != 442 {
|
||||
t.Fatalf("expected root id 442, got %d", result.ID)
|
||||
}
|
||||
|
||||
if len(result.WatchOrder) != 1 {
|
||||
t.Fatalf("expected 1 watch_order entry, got %d", len(result.WatchOrder))
|
||||
}
|
||||
|
||||
entry := result.WatchOrder[0]
|
||||
if entry.ID != 442 {
|
||||
t.Fatalf("expected entry id 442, got %d", entry.ID)
|
||||
}
|
||||
if entry.Type != "Movie" {
|
||||
t.Fatalf("expected type Movie, got %q", entry.Type)
|
||||
}
|
||||
if entry.Title != "Naruto Movie 1" {
|
||||
t.Fatalf("expected title Naruto Movie 1, got %q", entry.Title)
|
||||
}
|
||||
if entry.TitleAlt != "Naruto the Movie 1" {
|
||||
t.Fatalf("expected title_alt Naruto the Movie 1, got %q", entry.TitleAlt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchWatchOrder_NoRowsReturnsEmpty(t *testing.T) {
|
||||
server := testServer(testHTMLEmptyRows())
|
||||
defer server.Close()
|
||||
|
||||
url := server.URL + "/?/tools/watch_order/id/1535"
|
||||
result, err := FetchWatchOrder(context.Background(), &http.Client{Timeout: time.Second}, url)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if result.ID != 1535 {
|
||||
t.Fatalf("expected root id 1535, got %d", result.ID)
|
||||
}
|
||||
|
||||
if len(result.WatchOrder) != 0 {
|
||||
t.Fatalf("expected no entries, got %d", len(result.WatchOrder))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchWatchOrder_MissingMarkupFallsBackToProxy(t *testing.T) {
|
||||
proxyPayload := `Title: Jujutsu Kaisen / Watch Order
|
||||
URL Source: https://chiaki.site/?/tools/watch_order/id/40748
|
||||
|
||||
Markdown Content:
|
||||
Jujutsu Kaisen
|
||||
|
||||
Oct 3, 2020 – Mar 27, 2021 | TV | 24ep × 23min. | ★8.51 | [](https://myanimelist.net/anime/40748)
|
||||
Jujutsu Kaisen 0 Movie
|
||||
|
||||
Jujutsu Kaisen 0
|
||||
|
||||
Dec 24, 2021 | Movie | 1ep × 1hr. 44min. | ★8.36 | [](https://myanimelist.net/anime/48561)
|
||||
`
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasPrefix(r.URL.Path, "/http/") {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(proxyPayload))
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte("blocked"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
transport := http.DefaultTransport
|
||||
testClient := &http.Client{
|
||||
Timeout: time.Second,
|
||||
Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) {
|
||||
if strings.HasPrefix(request.URL.Host, "r.jina.ai") {
|
||||
proxyURL := server.URL + "/http/" + strings.TrimPrefix(request.URL.Path, "/")
|
||||
proxyRequest, err := http.NewRequestWithContext(request.Context(), request.Method, proxyURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return transport.RoundTrip(proxyRequest)
|
||||
}
|
||||
|
||||
blockedURL := server.URL + request.URL.Path
|
||||
blockedRequest, err := http.NewRequestWithContext(request.Context(), request.Method, blockedURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return transport.RoundTrip(blockedRequest)
|
||||
}),
|
||||
}
|
||||
|
||||
result, err := FetchWatchOrder(context.Background(), testClient, "https://chiaki.site/?/tools/watch_order/id/40748")
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if len(result.WatchOrder) != 2 {
|
||||
t.Fatalf("expected 2 proxy entries, got %d", len(result.WatchOrder))
|
||||
}
|
||||
|
||||
if result.WatchOrder[0].ID != 40748 || result.WatchOrder[0].Type != "TV" {
|
||||
t.Fatalf("unexpected first entry: %+v", result.WatchOrder[0])
|
||||
}
|
||||
|
||||
if result.WatchOrder[1].ID != 48561 || result.WatchOrder[1].Type != "Movie" {
|
||||
t.Fatalf("unexpected second entry: %+v", result.WatchOrder[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchWatchOrder_HTTPStatusErrorIncludesContext(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Server", "cloudflare")
|
||||
w.Header().Set("CF-Ray", "abc123")
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_, _ = w.Write([]byte("<html><body>access denied</body></html>"))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
url := server.URL + "/?/tools/watch_order/id/1"
|
||||
_, err := fetchDocument(context.Background(), &http.Client{Timeout: time.Second}, url)
|
||||
if err == nil {
|
||||
t.Fatalf("expected error, got nil")
|
||||
}
|
||||
|
||||
var statusError *HTTPStatusError
|
||||
if !errors.As(err, &statusError) {
|
||||
t.Fatalf("expected HTTPStatusError, got %T", err)
|
||||
}
|
||||
|
||||
if statusError.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("expected 403, got %d", statusError.StatusCode)
|
||||
}
|
||||
if statusError.CFRay != "abc123" {
|
||||
t.Fatalf("expected cf-ray abc123, got %q", statusError.CFRay)
|
||||
}
|
||||
if !strings.Contains(statusError.BodyPreview, "access denied") {
|
||||
t.Fatalf("expected body preview to include access denied, got %q", statusError.BodyPreview)
|
||||
}
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
return f(request)
|
||||
}
|
||||
Reference in New Issue
Block a user