core: harden watch-order relations
This commit is contained in:
184
internal/watchorder/watch_order.go
Normal file
184
internal/watchorder/watch_order.go
Normal file
@@ -0,0 +1,184 @@
|
||||
package watchorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "anime-relations-scraper/1.0 (+https://github.com/mkelvers/anime-relations)"
|
||||
|
||||
var idPattern = regexp.MustCompile(`/id/(\d+)`)
|
||||
|
||||
var ErrInvalidWatchOrderURL = errors.New("invalid watch order url")
|
||||
|
||||
type WatchOrderEntry struct {
|
||||
ID int `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
TitleAlt string `json:"title_alt,omitempty"`
|
||||
}
|
||||
|
||||
type WatchOrderResult struct {
|
||||
ID int `json:"id"`
|
||||
WatchOrder []WatchOrderEntry `json:"watch_order"`
|
||||
}
|
||||
|
||||
type watchOrderRow struct {
|
||||
id int
|
||||
typeID int
|
||||
title string
|
||||
alternativeTitle string
|
||||
}
|
||||
|
||||
func parseRootID(url string) (int, error) {
|
||||
match := idPattern.FindStringSubmatch(url)
|
||||
if len(match) != 2 {
|
||||
return 0, ErrInvalidWatchOrderURL
|
||||
}
|
||||
|
||||
id, err := strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
return 0, ErrInvalidWatchOrderURL
|
||||
}
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func fetchDocument(ctx context.Context, httpClient *http.Client, url string) (*goquery.Document, error) {
|
||||
client := httpClient
|
||||
if client == nil {
|
||||
client = http.DefaultClient
|
||||
}
|
||||
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
request.Header.Set("User-Agent", defaultUserAgent)
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request failed: %w", err)
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("unexpected status code: %d", response.StatusCode)
|
||||
}
|
||||
|
||||
document, err := goquery.NewDocumentFromReader(response.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse html: %w", err)
|
||||
}
|
||||
|
||||
return document, nil
|
||||
}
|
||||
|
||||
func extractTypeLabelsByID(doc *goquery.Document) map[int]string {
|
||||
typeLabels := make(map[int]string)
|
||||
|
||||
doc.Find("#wo_type_filter label").Each(func(_ int, selection *goquery.Selection) {
|
||||
input := selection.Find("input[type='checkbox']")
|
||||
rawID, exists := input.Attr("value")
|
||||
if !exists {
|
||||
return
|
||||
}
|
||||
|
||||
typeID, err := strconv.Atoi(strings.TrimSpace(rawID))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
label := strings.TrimSpace(selection.Text())
|
||||
if label == "" {
|
||||
return
|
||||
}
|
||||
|
||||
typeLabels[typeID] = label
|
||||
})
|
||||
|
||||
return typeLabels
|
||||
}
|
||||
|
||||
func parseAttrInt(selection *goquery.Selection, attrName string) (int, bool) {
|
||||
rawValue, exists := selection.Attr(attrName)
|
||||
if !exists {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
value, err := strconv.Atoi(strings.TrimSpace(rawValue))
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
return value, true
|
||||
}
|
||||
|
||||
func extractRows(doc *goquery.Document) []watchOrderRow {
|
||||
rows := make([]watchOrderRow, 0)
|
||||
|
||||
doc.Find("tr[data-id]").Each(func(_ int, selection *goquery.Selection) {
|
||||
id, ok := parseAttrInt(selection, "data-id")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
typeID, ok := parseAttrInt(selection, "data-type")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(selection.Find(".wo_title").First().Text())
|
||||
alternativeTitle := strings.TrimSpace(selection.Find(".uk-text-small").First().Text())
|
||||
|
||||
rows = append(rows, watchOrderRow{
|
||||
id: id,
|
||||
typeID: typeID,
|
||||
title: title,
|
||||
alternativeTitle: alternativeTitle,
|
||||
})
|
||||
})
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
func FetchWatchOrder(ctx context.Context, httpClient *http.Client, url string) (WatchOrderResult, error) {
|
||||
rootID, err := parseRootID(url)
|
||||
if err != nil {
|
||||
return WatchOrderResult{}, err
|
||||
}
|
||||
|
||||
doc, err := fetchDocument(ctx, httpClient, url)
|
||||
if err != nil {
|
||||
return WatchOrderResult{}, err
|
||||
}
|
||||
|
||||
rows := extractRows(doc)
|
||||
if len(rows) == 0 {
|
||||
return WatchOrderResult{ID: rootID, WatchOrder: []WatchOrderEntry{}}, nil
|
||||
}
|
||||
|
||||
typeByID := extractTypeLabelsByID(doc)
|
||||
|
||||
entries := make([]WatchOrderEntry, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
typeName := strings.TrimSpace(typeByID[row.typeID])
|
||||
|
||||
entries = append(entries, WatchOrderEntry{
|
||||
ID: row.id,
|
||||
Type: typeName,
|
||||
Title: row.title,
|
||||
TitleAlt: row.alternativeTitle,
|
||||
})
|
||||
}
|
||||
|
||||
return WatchOrderResult{ID: rootID, WatchOrder: entries}, nil
|
||||
}
|
||||
105
internal/watchorder/watch_order_test.go
Normal file
105
internal/watchorder/watch_order_test.go
Normal file
@@ -0,0 +1,105 @@
|
||||
package watchorder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testServer(body string) *httptest.Server {
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(body))
|
||||
})
|
||||
|
||||
return httptest.NewServer(handler)
|
||||
}
|
||||
|
||||
func testHTMLWithMetadata() string {
|
||||
return `
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<div id="wo_type_filter">
|
||||
<label><input type="checkbox" value="1" checked> TV</label>
|
||||
<label><input type="checkbox" value="3" checked> Movie</label>
|
||||
</div>
|
||||
<table id="wo_list">
|
||||
<tr data-id="442" data-anilist-id="442" data-type="3">
|
||||
<td>
|
||||
<span class="wo_title">Naruto Movie 1</span>
|
||||
<span class="uk-text-small">Naruto the Movie 1</span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
|
||||
func testHTMLEmptyRows() string {
|
||||
return `
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<div id="wo_type_filter">
|
||||
<label><input type="checkbox" value="1" checked> TV</label>
|
||||
<label><input type="checkbox" value="3" checked> Movie</label>
|
||||
</div>
|
||||
<table id="wo_list"></table>
|
||||
</body>
|
||||
</html>`
|
||||
}
|
||||
|
||||
func TestFetchWatchOrder_OutputShape(t *testing.T) {
|
||||
server := testServer(testHTMLWithMetadata())
|
||||
defer server.Close()
|
||||
|
||||
url := server.URL + "/?/tools/watch_order/id/442"
|
||||
result, err := FetchWatchOrder(context.Background(), &http.Client{Timeout: time.Second}, url)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if result.ID != 442 {
|
||||
t.Fatalf("expected root id 442, got %d", result.ID)
|
||||
}
|
||||
|
||||
if len(result.WatchOrder) != 1 {
|
||||
t.Fatalf("expected 1 watch_order entry, got %d", len(result.WatchOrder))
|
||||
}
|
||||
|
||||
entry := result.WatchOrder[0]
|
||||
if entry.ID != 442 {
|
||||
t.Fatalf("expected entry id 442, got %d", entry.ID)
|
||||
}
|
||||
if entry.Type != "Movie" {
|
||||
t.Fatalf("expected type Movie, got %q", entry.Type)
|
||||
}
|
||||
if entry.Title != "Naruto Movie 1" {
|
||||
t.Fatalf("expected title Naruto Movie 1, got %q", entry.Title)
|
||||
}
|
||||
if entry.TitleAlt != "Naruto the Movie 1" {
|
||||
t.Fatalf("expected title_alt Naruto the Movie 1, got %q", entry.TitleAlt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchWatchOrder_NoRowsReturnsEmpty(t *testing.T) {
|
||||
server := testServer(testHTMLEmptyRows())
|
||||
defer server.Close()
|
||||
|
||||
url := server.URL + "/?/tools/watch_order/id/1535"
|
||||
result, err := FetchWatchOrder(context.Background(), &http.Client{Timeout: time.Second}, url)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
|
||||
if result.ID != 1535 {
|
||||
t.Fatalf("expected root id 1535, got %d", result.ID)
|
||||
}
|
||||
|
||||
if len(result.WatchOrder) != 0 {
|
||||
t.Fatalf("expected no entries, got %d", len(result.WatchOrder))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user