diff --git a/readme.md b/readme.md index 3535823..ea57f36 100644 --- a/readme.md +++ b/readme.md @@ -32,6 +32,11 @@ To Be Played is split into three repositories: - **Playtime history**: one row per game and day, for the activity view and the "most played in the last two weeks" dashboard banner. - **Wishlist**: synced from Steam with store price and discount. +- **Broken image repair**: the frontend reports a Steam cover or achievement icon that does not load. At 01:30 + (Europe/Berlin) a job checks up to 200 reports. A cover that loads again is left alone; else the store's hashed + header picture (`IStoreBrowseService/GetItems` with assets) goes to `steam_game.image_url`. When a few achievement + icons do not load, the schema comes fresh from Steam; an empty answer keeps the stored one. A report that cannot be repaired is tried again on the next report, at most 3 times, + then after 30 days. - **Time to beat**: story, extras and completionist hours from the IGDB API. Optional: without credentials the feature is off and the rest works. - **Public profile data**: totals, status counts, games in progress and finished games with rating, perfect games @@ -57,6 +62,7 @@ accepted only by `GET /api/public/profiles/library` in the `X-Profile-Ticket` he | `PUT` | `/api/library/games/{appid}/backlog` | JWT | | `PUT` | `/api/library/games/{appid}/status` (status only, keeps rating, notes, priority) | JWT | | `POST` | `/api/library/sync` | JWT | +| `POST` | `/api/library/broken-images` (`{appid, kind}`, kind `COVER` or `ACHIEVEMENT_ICONS`; `204`) | JWT | | `GET` | `/api/library/picker?maxMinutes=&mood=&count=&exclude=` | JWT | | `GET` | `/api/library/achievements/summary` | JWT | | `GET` | `/api/library/sync-status`, `/summary`, `/activity`, `/wishlist`, `/steam-profile`, `/games-count` | JWT | @@ -97,6 +103,7 @@ flowchart LR Browser -->|"tbp_access cookie"| Security Security -->|"JWKS by kid"| Auth["ToBePlayed-Auth"] Steam --> SteamApi["Steam Web API"] + Steam -->|"HEAD"| SteamCdn["Steam CDN"] Igdb --> IgdbApi["IGDB API"] Igdb -->|"OAuth token"| Twitch["Twitch"] Speedrun --> SpeedrunApi["speedrun.com API"] @@ -108,7 +115,7 @@ flowchart LR |---|---| | `library` | Sync, backlog, manual games, playtime history, wishlist, achievements, public profile data | | `stats` | Public numbers for the landing page | -| `steam_adapter` | Steam Web API client | +| `steam_adapter` | Steam Web API client, availability check of Steam CDN pictures | | `igdb_adapter` | IGDB client (Twitch OAuth): time to beat, game tags for moods, release years, game search | | `speedrun_adapter` | speedrun.com client: game search by name, verified runs | @@ -142,6 +149,9 @@ Inside a module: `api` (controllers, DTOs), `service` (business logic), `domain` The frontend polls `GET /api/library/sync-status` until the run ends. +At 01:30 `ImageRepairService` repairs the pictures reported through `POST /api/library/broken-images` (table +`broken_image`, one row per game and kind). Reports only count for Steam games the user owns. + ### Manual games next to Steam games A game the user adds by hand comes from IGDB (`ManualGameService`). Steam games and manual games share diff --git a/src/main/java/core_service/modules/library/api/LibraryController.java b/src/main/java/core_service/modules/library/api/LibraryController.java index cf00b3a..efb8802 100644 --- a/src/main/java/core_service/modules/library/api/LibraryController.java +++ b/src/main/java/core_service/modules/library/api/LibraryController.java @@ -3,6 +3,7 @@ package core_service.modules.library.api; import core_service.modules.library.api.dto.AchievementSummaryDto; import core_service.modules.library.api.dto.ActivityDto; import core_service.modules.library.api.dto.BacklogEntryDto; +import core_service.modules.library.api.dto.BrokenImageReport; import core_service.modules.library.api.dto.GameDto; import core_service.modules.library.api.dto.LibrarySummaryDto; import core_service.modules.library.api.dto.PageDto; @@ -15,6 +16,7 @@ import core_service.modules.library.domain.GameStatus; import core_service.modules.library.domain.Mood; import core_service.modules.library.service.BacklogService; import core_service.modules.library.service.GameDetailService; +import core_service.modules.library.service.ImageRepairService; import core_service.modules.library.service.LibraryQueryService; import core_service.modules.library.service.LibrarySyncService; import core_service.modules.library.service.MoodService; @@ -64,6 +66,7 @@ public class LibraryController { private final SteamProfileService steamProfileService; private final MoodService moodService; private final SpeedrunService speedrunService; + private final ImageRepairService imageRepairService; @GetMapping("/games-count") public ResponseEntity getGamesCount(@AuthenticationPrincipal UserPrincipal principal) { @@ -139,6 +142,16 @@ public class LibraryController { .body(Map.of("code", "GAME_NOT_FOUND", "message", "Game not in user library"))); } + /** + * 204 also when the report is dropped, so the answer says nothing about the library. + */ + @PostMapping("/broken-images") + public ResponseEntity reportBrokenImage(@AuthenticationPrincipal UserPrincipal principal, + @Valid @RequestBody BrokenImageReport report) { + imageRepairService.report(principal.getIdAsLong(), report.appid(), report.kind()); + return ResponseEntity.noContent().build(); + } + @PutMapping("/games/{appid}/backlog") public ResponseEntity updateBacklog(@AuthenticationPrincipal UserPrincipal principal, @PathVariable("appid") Long appid, diff --git a/src/main/java/core_service/modules/library/api/dto/AchievementSummaryDto.java b/src/main/java/core_service/modules/library/api/dto/AchievementSummaryDto.java index fb433ab..b6448d6 100644 --- a/src/main/java/core_service/modules/library/api/dto/AchievementSummaryDto.java +++ b/src/main/java/core_service/modules/library/api/dto/AchievementSummaryDto.java @@ -10,7 +10,8 @@ import java.util.List; * @param averageCompletion mean completion in percent over the started games; null when none is started * @param pendingGames owned games the achievement sync has not fetched yet * @param hiddenGames games whose achievements Steam hides (private profile or stats) - * @param closest up to 5 started games that are not complete, closest to 100 % first + * @param closest up to 5 started games that are not complete, closest to 100 % first; {@code imageUrl} see + * {@link GameDto#imageUrl()} */ public record AchievementSummaryDto( int unlocked, @@ -22,5 +23,5 @@ public record AchievementSummaryDto( List closest ) { - public record Game(Long appid, String name, int unlocked, int total) {} + public record Game(Long appid, String name, int unlocked, int total, String imageUrl) {} } diff --git a/src/main/java/core_service/modules/library/api/dto/BrokenImageReport.java b/src/main/java/core_service/modules/library/api/dto/BrokenImageReport.java new file mode 100644 index 0000000..c597ea6 --- /dev/null +++ b/src/main/java/core_service/modules/library/api/dto/BrokenImageReport.java @@ -0,0 +1,6 @@ +package core_service.modules.library.api.dto; + +import core_service.modules.library.domain.ImageKind; +import jakarta.validation.constraints.NotNull; + +public record BrokenImageReport(@NotNull Long appid, @NotNull ImageKind kind) {} diff --git a/src/main/java/core_service/modules/library/api/dto/GameDto.java b/src/main/java/core_service/modules/library/api/dto/GameDto.java index c5d0f4e..d67980e 100644 --- a/src/main/java/core_service/modules/library/api/dto/GameDto.java +++ b/src/main/java/core_service/modules/library/api/dto/GameDto.java @@ -9,7 +9,7 @@ import core_service.modules.library.domain.GameStatus; * @param achievementsTotal null when the game has no achievements or they were not fetched yet * @param achievementsUnlocked null when not fetched yet or hidden by Steam * @param platform where the user owns a game added by hand, e.g. "PS5"; null for Steam games - * @param imageUrl picture for a game that has no Steam art; null means the Steam CDN has one + * @param imageUrl a game added by hand from IGDB, or a repaired Steam header; null means the fixed Steam CDN header */ public record GameDto( Long appid, diff --git a/src/main/java/core_service/modules/library/api/dto/RecentGameDto.java b/src/main/java/core_service/modules/library/api/dto/RecentGameDto.java index 1517c6c..7443120 100644 --- a/src/main/java/core_service/modules/library/api/dto/RecentGameDto.java +++ b/src/main/java/core_service/modules/library/api/dto/RecentGameDto.java @@ -4,11 +4,13 @@ import core_service.modules.library.domain.GameStatus; /** * @param playtime2Weeks Steam's playtime of the last two weeks, in minutes + * @param imageUrl see {@link GameDto#imageUrl()} */ public record RecentGameDto( Long appid, String name, int playtimeMinutes, int playtime2Weeks, - GameStatus status + GameStatus status, + String imageUrl ) {} diff --git a/src/main/java/core_service/modules/library/domain/BrokenImage.java b/src/main/java/core_service/modules/library/domain/BrokenImage.java new file mode 100644 index 0000000..0cc0921 --- /dev/null +++ b/src/main/java/core_service/modules/library/domain/BrokenImage.java @@ -0,0 +1,48 @@ +package core_service.modules.library.domain; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Id; +import jakarta.persistence.IdClass; +import jakarta.persistence.Table; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.time.Instant; + +@Entity +@Table(name = "broken_image") +@IdClass(BrokenImageId.class) +@Getter +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class BrokenImage { + + @Id + @Column(name = "appid") + private Long appid; + + @Id + @Enumerated(EnumType.STRING) + @Column(name = "kind", length = 24) + private ImageKind kind; + + @Column(name = "reported_at", nullable = false) + private Instant reportedAt; + + @Column(name = "checked_at") + private Instant checkedAt; + + @Column(name = "attempts", nullable = false) + private int attempts; + + public BrokenImage(Long appid, ImageKind kind, Instant reportedAt, Instant checkedAt, int attempts) { + this.appid = appid; + this.kind = kind; + this.reportedAt = reportedAt; + this.checkedAt = checkedAt; + this.attempts = attempts; + } +} diff --git a/src/main/java/core_service/modules/library/domain/BrokenImageId.java b/src/main/java/core_service/modules/library/domain/BrokenImageId.java new file mode 100644 index 0000000..5494213 --- /dev/null +++ b/src/main/java/core_service/modules/library/domain/BrokenImageId.java @@ -0,0 +1,19 @@ +package core_service.modules.library.domain; + +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode +public class BrokenImageId implements Serializable { + private Long appid; + private ImageKind kind; +} diff --git a/src/main/java/core_service/modules/library/domain/ImageKind.java b/src/main/java/core_service/modules/library/domain/ImageKind.java new file mode 100644 index 0000000..3cd5892 --- /dev/null +++ b/src/main/java/core_service/modules/library/domain/ImageKind.java @@ -0,0 +1,8 @@ +package core_service.modules.library.domain; + +public enum ImageKind { + // The header picture, also used as the cover. + COVER, + // Any achievement icon of the game: they all come from one schema. + ACHIEVEMENT_ICONS +} diff --git a/src/main/java/core_service/modules/library/repo/BrokenImageRepository.java b/src/main/java/core_service/modules/library/repo/BrokenImageRepository.java new file mode 100644 index 0000000..7c58a47 --- /dev/null +++ b/src/main/java/core_service/modules/library/repo/BrokenImageRepository.java @@ -0,0 +1,52 @@ +package core_service.modules.library.repo; + +import core_service.modules.library.domain.BrokenImage; +import core_service.modules.library.domain.BrokenImageId; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; + +import java.time.Instant; +import java.util.List; + +public interface BrokenImageRepository extends JpaRepository { + + @Modifying + @Transactional + @Query(value = "insert into broken_image (appid, kind, reported_at, attempts) values (:appid, :kind, :reportedAt, 0) " + + "on conflict (appid, kind) do update set reported_at = excluded.reported_at", + nativeQuery = true) + void report(@Param("appid") Long appid, @Param("kind") String kind, @Param("reportedAt") Instant reportedAt); + + @Modifying + @Transactional + @Query(value = "delete from broken_image where appid = :appid and kind = :kind and reported_at <= :reportedAt", + nativeQuery = true) + void deleteReportedUpTo(@Param("appid") Long appid, @Param("kind") String kind, @Param("reportedAt") Instant reportedAt); + + /** + * Rows reported after their last check. Never checked rows come first, then the oldest check: each report moves + * {@code reported_at}, so it cannot order the queue. A row out of attempts waits for {@code retryBefore}. + */ + @Query(value = "select * from broken_image " + + "where (checked_at is null or reported_at > checked_at) " + + "and (attempts < :maxAttempts or checked_at < :retryBefore) " + + "order by checked_at nulls first, reported_at limit :limit", + nativeQuery = true) + List findDue(@Param("maxAttempts") int maxAttempts, @Param("retryBefore") Instant retryBefore, + @Param("limit") int limit); + + /** + * One failed repair. A row that waited out its retry time counts from 1 again. + */ + @Modifying + @Transactional + @Query(value = "update broken_image set checked_at = :checkedAt, " + + "attempts = case when attempts >= :maxAttempts then 1 else attempts + 1 end " + + "where appid = :appid and kind = :kind", + nativeQuery = true) + void markChecked(@Param("appid") Long appid, @Param("kind") String kind, @Param("checkedAt") Instant checkedAt, + @Param("maxAttempts") int maxAttempts); +} diff --git a/src/main/java/core_service/modules/library/repo/SteamGameRepository.java b/src/main/java/core_service/modules/library/repo/SteamGameRepository.java index ad6a811..c3d6fe7 100644 --- a/src/main/java/core_service/modules/library/repo/SteamGameRepository.java +++ b/src/main/java/core_service/modules/library/repo/SteamGameRepository.java @@ -2,6 +2,15 @@ package core_service.modules.library.repo; import core_service.modules.library.domain.SteamGame; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; public interface SteamGameRepository extends JpaRepository { + + @Modifying + @Transactional + @Query("update SteamGame g set g.imageUrl = :imageUrl where g.appid = :appid") + void setImageUrl(@Param("appid") Long appid, @Param("imageUrl") String imageUrl); } diff --git a/src/main/java/core_service/modules/library/service/AchievementSyncService.java b/src/main/java/core_service/modules/library/service/AchievementSyncService.java index 33f4638..c1fa59b 100644 --- a/src/main/java/core_service/modules/library/service/AchievementSyncService.java +++ b/src/main/java/core_service/modules/library/service/AchievementSyncService.java @@ -125,6 +125,15 @@ public class AchievementSyncService { } } + /** + * Ignores the schema TTL, e.g. after an icon did not load. An empty answer changes nothing: storing it would + * hide the achievements for the whole TTL. + */ + public void refreshSchemaNow(Long appid) throws InterruptedException { + pause(); + refreshSchema(appid, true); + } + public void refreshUserIfStale(Long userId, String steamId, Long appid, int playtimeMinutes, Duration ttl) { Optional existing = userAchievementMetaRepository.findByUserIdAndAppid(userId, appid); if (existing.isEmpty() || existing.get().getRefreshedAt().isBefore(Instant.now().minus(ttl))) { @@ -192,11 +201,16 @@ public class AchievementSyncService { } private void refreshSchema(Long appid) { + refreshSchema(appid, false); + } + + private void refreshSchema(Long appid, boolean keepOnEmpty) { Optional schema = steamApiService.getSchemaForGame(appid); List achievements = schema .map(SteamSchemaForGameResponse.Game::availableGameStats) .map(SteamSchemaForGameResponse.AvailableGameStats::achievements) .orElse(List.of()); + if (keepOnEmpty && achievements.isEmpty()) return; writeLocked(0L, appid, () -> { if (!achievements.isEmpty()) { schemaRepository.deleteByAppid(appid); diff --git a/src/main/java/core_service/modules/library/service/ImageRepairService.java b/src/main/java/core_service/modules/library/service/ImageRepairService.java new file mode 100644 index 0000000..d94c199 --- /dev/null +++ b/src/main/java/core_service/modules/library/service/ImageRepairService.java @@ -0,0 +1,207 @@ +package core_service.modules.library.service; + +import core_service.modules.library.domain.BrokenImage; +import core_service.modules.library.domain.GameAchievementSchema; +import core_service.modules.library.domain.GameAchievementSchemaMeta; +import core_service.modules.library.domain.GameKeys; +import core_service.modules.library.domain.ImageKind; +import core_service.modules.library.domain.SteamGame; +import core_service.modules.library.repo.BrokenImageRepository; +import core_service.modules.library.repo.GameAchievementSchemaMetaRepository; +import core_service.modules.library.repo.GameAchievementSchemaRepository; +import core_service.modules.library.repo.SteamGameRepository; +import core_service.modules.library.repo.UserGameRepository; +import core_service.modules.steam_adapter.service.SteamApiService; +import core_service.modules.steam_adapter.service.SteamCdnService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Repairs the Steam pictures that browsers reported as broken, at night and only those. A report is only a hint: + * the job checks the picture itself first, so a false report (ad blocker, bad network) changes nothing. + */ +@Service +@Slf4j +public class ImageRepairService { + + private final BrokenImageRepository brokenImageRepository; + private final UserGameRepository userGameRepository; + private final SteamGameRepository steamGameRepository; + private final GameAchievementSchemaRepository schemaRepository; + private final GameAchievementSchemaMetaRepository schemaMetaRepository; + private final AchievementSyncService achievementSyncService; + private final SteamApiService steamApiService; + private final SteamCdnService steamCdnService; + private final String steamImageBaseUrl; + private final int batchSize; + private final int maxAttempts; + private final Duration retryAfter; + // Icons of one game change their hashes together, so a few checks tell a false report. + static final int ICONS_TO_CHECK = 5; + + public ImageRepairService(BrokenImageRepository brokenImageRepository, + UserGameRepository userGameRepository, + SteamGameRepository steamGameRepository, + GameAchievementSchemaRepository schemaRepository, + GameAchievementSchemaMetaRepository schemaMetaRepository, + AchievementSyncService achievementSyncService, + SteamApiService steamApiService, + SteamCdnService steamCdnService, + @Value("${app.steam.imageService-url}") String steamImageBaseUrl, + @Value("${app.library.image-repair-batch:200}") int batchSize, + @Value("${app.library.image-repair-max-attempts:3}") int maxAttempts, + @Value("${app.library.image-repair-retry-days:30}") long retryDays) { + this.brokenImageRepository = brokenImageRepository; + this.userGameRepository = userGameRepository; + this.steamGameRepository = steamGameRepository; + this.schemaRepository = schemaRepository; + this.schemaMetaRepository = schemaMetaRepository; + this.achievementSyncService = achievementSyncService; + this.steamApiService = steamApiService; + this.steamCdnService = steamCdnService; + this.steamImageBaseUrl = steamImageBaseUrl; + this.batchSize = batchSize; + this.maxAttempts = maxAttempts; + this.retryAfter = Duration.ofDays(retryDays); + } + + /** + * Only for a Steam game the user owns; anything else is dropped. + */ + public void report(Long userId, Long appid, ImageKind kind) { + if (!GameKeys.isSteam(appid) || userGameRepository.findActiveAppids(userId, List.of(appid)).isEmpty()) { + return; + } + brokenImageRepository.report(appid, kind.name(), Instant.now()); + } + + @Scheduled(cron = "${app.library.image-repair-cron:0 30 1 * * *}", zone = "${app.library.zone:Europe/Berlin}") + public void repairNightly() { + Instant startedAt = Instant.now(); + List due = brokenImageRepository.findDue(maxAttempts, startedAt.minus(retryAfter), batchSize); + if (due.isEmpty()) return; + log.info("Repairing {} reported images", due.size()); + List covers = due.stream().filter(row -> row.getKind() == ImageKind.COVER).toList(); + try { + repairCovers(covers, startedAt); + } catch (Exception e) { + log.warn("Cover repair failed", e); + covers.forEach(row -> markCheckedQuietly(row, startedAt)); + } + for (BrokenImage row : due) { + if (row.getKind() != ImageKind.ACHIEVEMENT_ICONS) continue; + try { + repairAchievementIcons(row, startedAt); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } catch (Exception e) { + log.warn("Achievement icon repair failed for app {}", row.getAppid(), e); + markCheckedQuietly(row, startedAt); + } + } + } + + private void repairCovers(List rows, Instant startedAt) { + if (rows.isEmpty()) return; + Map games = steamGameRepository.findAllById(rows.stream().map(BrokenImage::getAppid).toList()) + .stream().collect(Collectors.toMap(SteamGame::getAppid, Function.identity())); + List broken = new ArrayList<>(); + for (BrokenImage row : rows) { + SteamGame game = games.get(row.getAppid()); + String current = game != null && game.getImageUrl() != null + ? game.getImageUrl() : steamImageBaseUrl + row.getAppid() + "/header.jpg"; + if (steamCdnService.isAvailable(current)) { + delete(row); + } else { + broken.add(row); + } + } + if (broken.isEmpty()) return; + Map storeHeaders = steamApiService.getStoreHeaderUrls(broken.stream().map(BrokenImage::getAppid).toList()); + for (BrokenImage row : broken) { + try { + String url = storeHeaders.get(row.getAppid()); + if (url != null && steamCdnService.isAvailable(url)) { + steamGameRepository.setImageUrl(row.getAppid(), url); + delete(row); + } else { + markChecked(row, startedAt); + } + } catch (Exception e) { + log.warn("Cover repair failed for app {}", row.getAppid(), e); + markCheckedQuietly(row, startedAt); + } + } + } + + /** + * The icon URLs come from the schema. A schema refreshed after the report already has the new ones; else the + * repair asks Steam again and counts as done when any URL changed. + */ + private void repairAchievementIcons(BrokenImage row, Instant startedAt) throws InterruptedException { + Long appid = row.getAppid(); + Optional meta = schemaMetaRepository.findById(appid); + if (meta.isPresent() && meta.get().getRefreshedAt().isAfter(row.getReportedAt())) { + delete(row); + return; + } + Set before = iconUrls(appid); + List sample = before.stream().filter(Objects::nonNull).limit(ICONS_TO_CHECK).toList(); + if (!sample.isEmpty() && sample.stream().allMatch(steamCdnService::isAvailable)) { + delete(row); + return; + } + achievementSyncService.refreshSchemaNow(appid); + if (iconUrls(appid).equals(before)) { + markChecked(row, startedAt); + } else { + delete(row); + } + } + + private Set iconUrls(Long appid) { + Set urls = new LinkedHashSet<>(); + for (GameAchievementSchema achievement : schemaRepository.findByAppid(appid)) { + urls.add(achievement.getIconUrl()); + urls.add(achievement.getIconGrayUrl()); + } + return urls; + } + + /** + * Keeps a row that was reported again during the run. + */ + private void delete(BrokenImage row) { + brokenImageRepository.deleteReportedUpTo(row.getAppid(), row.getKind().name(), row.getReportedAt()); + } + + /** + * Checked at the start of the run, so a report during the run makes the row due again. + */ + private void markChecked(BrokenImage row, Instant startedAt) { + brokenImageRepository.markChecked(row.getAppid(), row.getKind().name(), startedAt, maxAttempts); + } + + private void markCheckedQuietly(BrokenImage row, Instant startedAt) { + try { + markChecked(row, startedAt); + } catch (Exception e) { + log.warn("Could not store the failed repair of app {}", row.getAppid(), e); + } + } +} diff --git a/src/main/java/core_service/modules/library/service/LibraryQueryService.java b/src/main/java/core_service/modules/library/service/LibraryQueryService.java index ca87a15..a80ba67 100644 --- a/src/main/java/core_service/modules/library/service/LibraryQueryService.java +++ b/src/main/java/core_service/modules/library/service/LibraryQueryService.java @@ -130,7 +130,8 @@ public class LibraryQueryService { closest.stream() .map(g -> new AchievementSummaryDto.Game(g.appid(), names.containsKey(g.appid()) ? names.get(g.appid()).getName() : null, - g.unlockedCapped(), g.total())) + g.unlockedCapped(), g.total(), + names.containsKey(g.appid()) ? names.get(g.appid()).getImageUrl() : null)) .toList()); } @@ -184,7 +185,8 @@ public class LibraryQueryService { meta.containsKey(ug.getAppid()) ? meta.get(ug.getAppid()).getName() : null, ug.getPlaytimeMinutes(), ug.getPlaytime2Weeks(), - ug.getStatus())) + ug.getStatus(), + meta.containsKey(ug.getAppid()) ? meta.get(ug.getAppid()).getImageUrl() : null)) .toList(); return new ActivityDto(perDay, total, recentGames); } diff --git a/src/main/java/core_service/modules/stats/api/dto/PublicStatsDto.java b/src/main/java/core_service/modules/stats/api/dto/PublicStatsDto.java index b054a96..d7ac363 100644 --- a/src/main/java/core_service/modules/stats/api/dto/PublicStatsDto.java +++ b/src/main/java/core_service/modules/stats/api/dto/PublicStatsDto.java @@ -16,7 +16,7 @@ public record PublicStatsDto( List topPlayedGames ) { /** - * @param imageUrl picture for a game that has no Steam art, see GameDto + * @param imageUrl see GameDto */ public record TopGame(Long appid, String name, String imgIconUrl, String imageUrl, long value) {} } diff --git a/src/main/java/core_service/modules/steam_adapter/api/dto/response/SteamStoreItemsResponse.java b/src/main/java/core_service/modules/steam_adapter/api/dto/response/SteamStoreItemsResponse.java index be0a7f1..1c3d9a0 100644 --- a/src/main/java/core_service/modules/steam_adapter/api/dto/response/SteamStoreItemsResponse.java +++ b/src/main/java/core_service/modules/steam_adapter/api/dto/response/SteamStoreItemsResponse.java @@ -18,13 +18,25 @@ public record SteamStoreItemsResponse( /** * @param success 1 for a known app * @param bestPurchaseOption null for an app without a price, e.g. not released yet + * @param assets only sent when the request asks for them */ @JsonIgnoreProperties(ignoreUnknown = true) public record StoreItem( @JsonProperty("appid") Long appid, @JsonProperty("success") Integer success, @JsonProperty("name") String name, - @JsonProperty("best_purchase_option") PurchaseOption bestPurchaseOption + @JsonProperty("best_purchase_option") PurchaseOption bestPurchaseOption, + @JsonProperty("assets") Assets assets + ) {} + + /** + * @param assetUrlFormat e.g. {@code steam/apps/620/${FILENAME}?t=1776125736} + * @param header e.g. {@code 7983574d464e6559ac7e24275727f73a8bcca1f3/header.jpg} + */ + @JsonIgnoreProperties(ignoreUnknown = true) + public record Assets( + @JsonProperty("asset_url_format") String assetUrlFormat, + @JsonProperty("header") String header ) {} @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/src/main/java/core_service/modules/steam_adapter/config/SteamClientConfig.java b/src/main/java/core_service/modules/steam_adapter/config/SteamClientConfig.java index 60c12a7..00bc836 100644 --- a/src/main/java/core_service/modules/steam_adapter/config/SteamClientConfig.java +++ b/src/main/java/core_service/modules/steam_adapter/config/SteamClientConfig.java @@ -3,8 +3,11 @@ package core_service.modules.steam_adapter.config; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.SimpleClientHttpRequestFactory; import org.springframework.web.client.RestClient; +import java.time.Duration; + @Configuration public class SteamClientConfig { @Bean @@ -13,4 +16,14 @@ public class SteamClientConfig { .baseUrl(baseUrl) .build(); } + + @Bean + public RestClient steamCdnRestClient() { + SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); + requestFactory.setConnectTimeout(Duration.ofSeconds(3)); + requestFactory.setReadTimeout(Duration.ofSeconds(10)); + return RestClient.builder() + .requestFactory(requestFactory) + .build(); + } } diff --git a/src/main/java/core_service/modules/steam_adapter/service/SteamApiService.java b/src/main/java/core_service/modules/steam_adapter/service/SteamApiService.java index 2a74d54..a869646 100644 --- a/src/main/java/core_service/modules/steam_adapter/service/SteamApiService.java +++ b/src/main/java/core_service/modules/steam_adapter/service/SteamApiService.java @@ -35,6 +35,9 @@ public class SteamApiService { private final RestClient steamRestClient; private static final int STORE_ITEMS_BATCH = 100; + private static final String ASSET_FILENAME = "${FILENAME}"; + // The cloudflare "shared" host answers 301 for these paths; akamai serves them. + private static final String STORE_ASSET_HOST = "https://shared.akamai.steamstatic.com/store_item_assets/"; @Value("${app.steam.api-key}") private String apiKey; @@ -215,10 +218,46 @@ public class SteamApiService { return result; } + /** + * The store's hashed header picture per app. It loads when {@code steam/apps//header.jpg} does not. + * Apps without one and failed batches are missing. + */ + @RateLimiter(name = "steamApi") + public Map getStoreHeaderUrls(List appids) { + Map result = new HashMap<>(); + for (int from = 0; from < appids.size(); from += STORE_ITEMS_BATCH) { + List batch = appids.subList(from, Math.min(from + STORE_ITEMS_BATCH, appids.size())); + for (SteamStoreItemsResponse.StoreItem item : fetchStoreItems(batch, "{\"include_assets\":true}")) { + SteamStoreItemsResponse.Assets assets = item.assets(); + if (assets != null && assets.assetUrlFormat() != null && assets.header() != null + && assets.assetUrlFormat().contains(ASSET_FILENAME)) { + result.put(item.appid(), STORE_ASSET_HOST + assets.assetUrlFormat().replace(ASSET_FILENAME, assets.header())); + } + } + } + return result; + } + private Map getStoreItemsBatch(List appids) { + Map items = new HashMap<>(); + for (SteamStoreItemsResponse.StoreItem item : fetchStoreItems(appids, "{}")) { + SteamStoreItemsResponse.PurchaseOption price = item.bestPurchaseOption(); + items.put(item.appid(), new StoreItem( + item.name(), + price != null ? price.formattedFinalPrice() : null, + price != null ? price.formattedOriginalPrice() : null, + price != null && price.discountPct() != null ? price.discountPct() : 0)); + } + return items; + } + + /** + * @return the known apps (success 1); empty when the request failed + */ + private List fetchStoreItems(List appids, String dataRequest) { String ids = appids.stream().map(id -> "{\"appid\":" + id + "}").collect(Collectors.joining(",")); String inputJson = "{\"ids\":[" + ids + "],\"context\":{\"language\":\"" + storeLanguage - + "\",\"country_code\":\"" + storeCountry + "\"},\"data_request\":{}}"; + + "\",\"country_code\":\"" + storeCountry + "\"},\"data_request\":" + dataRequest + "}"; try { SteamStoreItemsResponse response = steamRestClient.get() .uri(uriBuilder -> uriBuilder @@ -229,29 +268,21 @@ public class SteamApiService { .retrieve() .body(SteamStoreItemsResponse.class); if (response == null || response.response() == null || response.response().storeItems() == null) { - return Map.of(); + return List.of(); } - Map items = new HashMap<>(); - for (SteamStoreItemsResponse.StoreItem item : response.response().storeItems()) { - if (item.appid() == null || !Integer.valueOf(1).equals(item.success())) continue; - SteamStoreItemsResponse.PurchaseOption price = item.bestPurchaseOption(); - items.put(item.appid(), new StoreItem( - item.name(), - price != null ? price.formattedFinalPrice() : null, - price != null ? price.formattedOriginalPrice() : null, - price != null && price.discountPct() != null ? price.discountPct() : 0)); - } - return items; + return response.response().storeItems().stream() + .filter(item -> item.appid() != null && Integer.valueOf(1).equals(item.success())) + .toList(); } catch (Exception e) { log.error("Error communicating with Steam API for store items {}", appids, e); - return Map.of(); + return List.of(); } } /** * Per-game achievement schema (global, language-aware). * Returns empty when the game has no achievements or Steam answers 4xx, its final answer for delisted apps and - * tools. A server or network failure throws, so the caller keeps what it stored before. + * tools. A server or network failure, 408 or 429 throws, so the caller keeps what it stored before. */ @RateLimiter(name = "steamApi") @Retry(name = "steamApi") @@ -269,6 +300,9 @@ public class SteamApiService { return Optional.of(response.game()); } } catch (HttpClientErrorException e) { + if (e.getStatusCode().value() == 408 || e.getStatusCode().value() == 429) { + throw e; + } log.info("No achievement schema for appid {}: {}", appid, e.getStatusCode()); } return Optional.empty(); diff --git a/src/main/java/core_service/modules/steam_adapter/service/SteamCdnService.java b/src/main/java/core_service/modules/steam_adapter/service/SteamCdnService.java new file mode 100644 index 0000000..fc65f91 --- /dev/null +++ b/src/main/java/core_service/modules/steam_adapter/service/SteamCdnService.java @@ -0,0 +1,31 @@ +package core_service.modules.steam_adapter.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestClient; + +@Service +@RequiredArgsConstructor +@Slf4j +public class SteamCdnService { + + private final RestClient steamCdnRestClient; + + /** + * {@code HEAD} on a Steam CDN picture. A redirect counts as not available: the request does not follow it. + */ + public boolean isAvailable(String url) { + try { + return steamCdnRestClient.head() + .uri(url) + .retrieve() + .toBodilessEntity() + .getStatusCode() + .is2xxSuccessful(); + } catch (Exception e) { + log.debug("Steam CDN check failed for {}: {}", url, e.getMessage()); + return false; + } + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index aed3d2f..b4e616d 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -61,6 +61,11 @@ app: achievement-ttl-hours: 1 schema-ttl-days: 30 speedrun-ttl-days: 30 + # Reported broken pictures, repaired after the nightly sync. + image-repair-cron: "0 30 1 * * *" + image-repair-batch: 200 + image-repair-max-attempts: 3 + image-repair-retry-days: 30 resilience4j: retry: diff --git a/src/main/resources/db/migration/V11__broken_image.sql b/src/main/resources/db/migration/V11__broken_image.sql new file mode 100644 index 0000000..6737bc3 --- /dev/null +++ b/src/main/resources/db/migration/V11__broken_image.sql @@ -0,0 +1,9 @@ +-- Pictures a browser could not load. One row per game and kind; a new report only moves reported_at. +CREATE TABLE broken_image ( + appid BIGINT NOT NULL, + kind VARCHAR(24) NOT NULL, + reported_at TIMESTAMPTZ NOT NULL, + checked_at TIMESTAMPTZ, + attempts INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (appid, kind) +); diff --git a/src/test/java/core_service/modules/library/api/LibraryControllerAchievementsTest.java b/src/test/java/core_service/modules/library/api/LibraryControllerAchievementsTest.java index f71c324..3e84aaa 100644 --- a/src/test/java/core_service/modules/library/api/LibraryControllerAchievementsTest.java +++ b/src/test/java/core_service/modules/library/api/LibraryControllerAchievementsTest.java @@ -41,6 +41,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. class LibraryControllerAchievementsTest extends AbstractIntegrationTest { private static final long USER_ID = 1L; + private static final String REPAIRED_COVER = "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1/x/header.jpg"; @Autowired private MockMvc mockMvc; @@ -106,6 +107,9 @@ class LibraryControllerAchievementsTest extends AbstractIntegrationTest { game(70L, 8, null, true); // unlocks never fetched game(80L, 10, 12, true); // schema lost achievements, still perfect game(90L, 6, 0, true); // not started + SteamGame repaired = steamGameRepository.findById(20L).orElseThrow(); + repaired.setImageUrl(REPAIRED_COVER); + steamGameRepository.save(repaired); steamGameRepository.save(new SteamGame(95L, "Game 95", null, Instant.now())); UserGame removed = new UserGame(USER_ID, 95L, 60, Instant.now(), Instant.now()); removed.setRemovedAt(Instant.now()); @@ -128,7 +132,9 @@ class LibraryControllerAchievementsTest extends AbstractIntegrationTest { .andExpect(jsonPath("$.closest[0].name").value("Game 20")) .andExpect(jsonPath("$.closest[0].unlocked").value(3)) .andExpect(jsonPath("$.closest[0].total").value(4)) - .andExpect(jsonPath("$.closest[1].appid").value(30)); + .andExpect(jsonPath("$.closest[0].imageUrl").value(REPAIRED_COVER)) + .andExpect(jsonPath("$.closest[1].appid").value(30)) + .andExpect(jsonPath("$.closest[1].imageUrl").doesNotExist()); } @Test diff --git a/src/test/java/core_service/modules/library/api/LibraryControllerActivityTest.java b/src/test/java/core_service/modules/library/api/LibraryControllerActivityTest.java index a95a41e..3d62aac 100644 --- a/src/test/java/core_service/modules/library/api/LibraryControllerActivityTest.java +++ b/src/test/java/core_service/modules/library/api/LibraryControllerActivityTest.java @@ -30,6 +30,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. class LibraryControllerActivityTest extends AbstractIntegrationTest { private static final long USER_ID = 1L; + private static final String REPAIRED_COVER = "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1/x/header.jpg"; @Autowired private MockMvc mockMvc; @@ -52,9 +53,11 @@ class LibraryControllerActivityTest extends AbstractIntegrationTest { void seed() { userGameRepository.deleteAll(); jdbc.update("delete from user_game_playtime_day"); + SteamGame repairedPortal = new SteamGame(620L, "Portal 2", null, Instant.now()); + repairedPortal.setImageUrl(REPAIRED_COVER); steamGameRepository.saveAll(List.of( new SteamGame(220L, "Half-Life 2", null, Instant.now()), - new SteamGame(620L, "Portal 2", null, Instant.now()), + repairedPortal, new SteamGame(700L, "Dota 2", null, Instant.now()))); UserGame hl2 = new UserGame(USER_ID, 220L, 300, Instant.now(), Instant.now()); hl2.setPlaytime2Weeks(30); @@ -83,7 +86,9 @@ class LibraryControllerActivityTest extends AbstractIntegrationTest { .andExpect(jsonPath("$.recentGames[0].appid").value(620)) .andExpect(jsonPath("$.recentGames[0].name").value("Portal 2")) .andExpect(jsonPath("$.recentGames[0].playtime2Weeks").value(120)) - .andExpect(jsonPath("$.recentGames[1].appid").value(220)); + .andExpect(jsonPath("$.recentGames[0].imageUrl").value(REPAIRED_COVER)) + .andExpect(jsonPath("$.recentGames[1].appid").value(220)) + .andExpect(jsonPath("$.recentGames[1].imageUrl").doesNotExist()); } @Test diff --git a/src/test/java/core_service/modules/library/api/LibraryControllerBrokenImageTest.java b/src/test/java/core_service/modules/library/api/LibraryControllerBrokenImageTest.java new file mode 100644 index 0000000..8caf6bc --- /dev/null +++ b/src/test/java/core_service/modules/library/api/LibraryControllerBrokenImageTest.java @@ -0,0 +1,92 @@ +package core_service.modules.library.api; + +import core_service.configuration.security.ClientHeaderFilter; +import core_service.modules.library.domain.BrokenImageId; +import core_service.modules.library.domain.ImageKind; +import core_service.modules.library.domain.SteamGame; +import core_service.modules.library.domain.UserGame; +import core_service.modules.library.repo.BrokenImageRepository; +import core_service.modules.library.repo.SteamGameRepository; +import core_service.modules.library.repo.UserGameRepository; +import core_service.shared.test.AbstractIntegrationTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * The dev profile authenticates every request as user 1. + */ +@AutoConfigureMockMvc +class LibraryControllerBrokenImageTest extends AbstractIntegrationTest { + + private static final long USER_ID = 1L; + private static final long APPID = 220L; + + @Autowired + private MockMvc mockMvc; + + @Autowired + private SteamGameRepository steamGameRepository; + + @Autowired + private UserGameRepository userGameRepository; + + @Autowired + private BrokenImageRepository brokenImageRepository; + + @BeforeEach + void seed() { + brokenImageRepository.deleteAll(); + userGameRepository.deleteAll(); + steamGameRepository.save(new SteamGame(APPID, "Half-Life 2", null, Instant.now())); + userGameRepository.save(new UserGame(USER_ID, APPID, 0, Instant.now(), Instant.now())); + } + + @Test + void reportBrokenImage_StoresTheReport() throws Exception { + mockMvc.perform(post("/api/library/broken-images").header(ClientHeaderFilter.CLIENT_HEADER, "test") + .contentType(MediaType.APPLICATION_JSON).content("{\"appid\":220,\"kind\":\"ACHIEVEMENT_ICONS\"}")) + .andExpect(status().isNoContent()); + + assertThat(brokenImageRepository.findById(new BrokenImageId(APPID, ImageKind.ACHIEVEMENT_ICONS))).isPresent(); + } + + @Test + void reportBrokenImage_ForAGameNotOwnedAnswersTheSameAndStoresNothing() throws Exception { + mockMvc.perform(post("/api/library/broken-images").header(ClientHeaderFilter.CLIENT_HEADER, "test") + .contentType(MediaType.APPLICATION_JSON).content("{\"appid\":620,\"kind\":\"COVER\"}")) + .andExpect(status().isNoContent()); + + assertThat(brokenImageRepository.count()).isZero(); + } + + @ParameterizedTest + @ValueSource(strings = {"{\"appid\":220,\"kind\":\"BANNER\"}", "{\"appid\":220}", "{\"kind\":\"COVER\"}"}) + void reportBrokenImage_BadBodyIs400(String body) throws Exception { + mockMvc.perform(post("/api/library/broken-images").header(ClientHeaderFilter.CLIENT_HEADER, "test") + .contentType(MediaType.APPLICATION_JSON).content(body)) + .andExpect(status().isBadRequest()); + + assertThat(brokenImageRepository.count()).isZero(); + } + + @Test + void reportBrokenImage_WithoutClientHeaderIs403() throws Exception { + mockMvc.perform(post("/api/library/broken-images") + .contentType(MediaType.APPLICATION_JSON).content("{\"appid\":220,\"kind\":\"COVER\"}")) + .andExpect(status().isForbidden()); + + assertThat(brokenImageRepository.count()).isZero(); + } +} diff --git a/src/test/java/core_service/modules/library/repo/BrokenImageRepositoryTest.java b/src/test/java/core_service/modules/library/repo/BrokenImageRepositoryTest.java new file mode 100644 index 0000000..9ea303c --- /dev/null +++ b/src/test/java/core_service/modules/library/repo/BrokenImageRepositoryTest.java @@ -0,0 +1,117 @@ +package core_service.modules.library.repo; + +import core_service.modules.library.domain.BrokenImage; +import core_service.modules.library.domain.BrokenImageId; +import core_service.modules.library.domain.ImageKind; +import core_service.shared.test.AbstractPersistenceTest; +import jakarta.persistence.EntityManager; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class BrokenImageRepositoryTest extends AbstractPersistenceTest { + + private static final int MAX_ATTEMPTS = 3; + + @Autowired + private BrokenImageRepository brokenImageRepository; + + @Autowired + private EntityManager entityManager; + + private final Instant now = Instant.now().truncatedTo(ChronoUnit.MICROS); + + @BeforeEach + void clean() { + brokenImageRepository.deleteAll(); + } + + private List dueAppids(int limit) { + return brokenImageRepository.findDue(MAX_ATTEMPTS, now.minus(Duration.ofDays(30)), limit).stream() + .map(BrokenImage::getAppid).toList(); + } + + private BrokenImage stored(long appid, ImageKind kind) { + entityManager.clear(); + return brokenImageRepository.findById(new BrokenImageId(appid, kind)).orElseThrow(); + } + + @Test + void report_KeepsOneRowPerGameAndKindAndMovesTheReportTime() { + brokenImageRepository.report(620L, "COVER", now.minus(Duration.ofDays(1))); + brokenImageRepository.markChecked(620L, "COVER", now.minus(Duration.ofHours(12)), MAX_ATTEMPTS); + brokenImageRepository.report(620L, "COVER", now); + brokenImageRepository.report(620L, "ACHIEVEMENT_ICONS", now); + + assertThat(brokenImageRepository.count()).isEqualTo(2); + BrokenImage cover = stored(620L, ImageKind.COVER); + assertThat(cover.getReportedAt()).isEqualTo(now); + assertThat(cover.getAttempts()).isEqualTo(1); + } + + @Test + void findDue_OldestReportFirstAndOnlyUpToTheLimit() { + brokenImageRepository.report(620L, "COVER", now.minus(Duration.ofHours(1))); + brokenImageRepository.report(220L, "COVER", now.minus(Duration.ofHours(3))); + brokenImageRepository.report(400L, "COVER", now.minus(Duration.ofHours(2))); + + assertThat(dueAppids(2)).containsExactly(220L, 400L); + } + + @Test + void findDue_NeverCheckedRowsFirstThenTheLongestWaitingCheck() { + brokenImageRepository.report(620L, "COVER", now.minus(Duration.ofDays(5))); + brokenImageRepository.markChecked(620L, "COVER", now.minus(Duration.ofDays(4)), MAX_ATTEMPTS); + brokenImageRepository.report(620L, "COVER", now); + brokenImageRepository.report(220L, "COVER", now.minus(Duration.ofDays(3))); + brokenImageRepository.markChecked(220L, "COVER", now.minus(Duration.ofDays(2)), MAX_ATTEMPTS); + brokenImageRepository.report(220L, "COVER", now.minus(Duration.ofDays(1))); + brokenImageRepository.report(400L, "COVER", now); + + assertThat(dueAppids(10)).containsExactly(400L, 620L, 220L); + } + + @Test + void deleteReportedUpTo_KeepsARowReportedAgain() { + brokenImageRepository.report(620L, "COVER", now.minus(Duration.ofHours(1))); + brokenImageRepository.report(620L, "COVER", now); + + brokenImageRepository.deleteReportedUpTo(620L, "COVER", now.minus(Duration.ofHours(1))); + assertThat(brokenImageRepository.count()).isEqualTo(1); + + brokenImageRepository.deleteReportedUpTo(620L, "COVER", now); + assertThat(brokenImageRepository.count()).isZero(); + } + + @Test + void findDue_ACheckedRowWaitsForTheNextReport() { + brokenImageRepository.report(620L, "COVER", now.minus(Duration.ofHours(2))); + brokenImageRepository.markChecked(620L, "COVER", now.minus(Duration.ofHours(1)), MAX_ATTEMPTS); + assertThat(dueAppids(10)).isEmpty(); + + brokenImageRepository.report(620L, "COVER", now); + assertThat(dueAppids(10)).containsExactly(620L); + } + + @Test + void findDue_AfterTheLastAttemptARowWaitsForTheRetryTimeAndStartsAgain() { + brokenImageRepository.report(620L, "COVER", now.minus(Duration.ofDays(40))); + for (int i = 0; i < MAX_ATTEMPTS; i++) { + brokenImageRepository.markChecked(620L, "COVER", now.minus(Duration.ofDays(10)), MAX_ATTEMPTS); + } + brokenImageRepository.report(620L, "COVER", now); + assertThat(dueAppids(10)).isEmpty(); + + brokenImageRepository.markChecked(620L, "COVER", now.minus(Duration.ofDays(31)), MAX_ATTEMPTS); + assertThat(stored(620L, ImageKind.COVER).getAttempts()).isEqualTo(1); + brokenImageRepository.report(620L, "COVER", now); + assertThat(dueAppids(10)).containsExactly(620L); + } +} diff --git a/src/test/java/core_service/modules/library/service/AchievementSyncServiceTest.java b/src/test/java/core_service/modules/library/service/AchievementSyncServiceTest.java index 835c175..197178c 100644 --- a/src/test/java/core_service/modules/library/service/AchievementSyncServiceTest.java +++ b/src/test/java/core_service/modules/library/service/AchievementSyncServiceTest.java @@ -173,6 +173,16 @@ class AchievementSyncServiceTest extends AbstractUnitTest { verify(steamApiService, never()).getPlayerAchievements(anyString(), anyLong()); } + @Test + void refreshSchemaNow_EmptyAnswer_KeepsTheStoredSchema() throws Exception { + when(steamApiService.getSchemaForGame(620L)).thenReturn(Optional.empty()); + + service.refreshSchemaNow(620L); + + verify(schemaMetaRepository, never()).save(any()); + verify(schemaRepository, never()).deleteByAppid(anyLong()); + } + @Test void syncLibrary_OldSchema_IsFetchedAgain() { GameAchievementSchemaMeta old = new GameAchievementSchemaMeta(620L, 10, true, Instant.now().minus(Duration.ofDays(31))); diff --git a/src/test/java/core_service/modules/library/service/ImageRepairServiceTest.java b/src/test/java/core_service/modules/library/service/ImageRepairServiceTest.java new file mode 100644 index 0000000..5c9f63d --- /dev/null +++ b/src/test/java/core_service/modules/library/service/ImageRepairServiceTest.java @@ -0,0 +1,284 @@ +package core_service.modules.library.service; + +import core_service.modules.library.domain.BrokenImage; +import core_service.modules.library.domain.GameAchievementSchema; +import core_service.modules.library.domain.GameAchievementSchemaMeta; +import core_service.modules.library.domain.ImageKind; +import core_service.modules.library.domain.SteamGame; +import core_service.modules.library.repo.BrokenImageRepository; +import core_service.modules.library.repo.GameAchievementSchemaMetaRepository; +import core_service.modules.library.repo.GameAchievementSchemaRepository; +import core_service.modules.library.repo.SteamGameRepository; +import core_service.modules.library.repo.UserGameRepository; +import core_service.modules.steam_adapter.service.SteamApiService; +import core_service.modules.steam_adapter.service.SteamCdnService; +import core_service.shared.test.AbstractUnitTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +class ImageRepairServiceTest extends AbstractUnitTest { + + private static final String BASE = "https://cdn.cloudflare.steamstatic.com/steam/apps/"; + private static final String FIXED_620 = BASE + "620/header.jpg"; + private static final String HASHED_620 = "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/620/abc/header.jpg?t=1"; + + @Mock + private BrokenImageRepository brokenImageRepository; + @Mock + private UserGameRepository userGameRepository; + @Mock + private SteamGameRepository steamGameRepository; + @Mock + private GameAchievementSchemaRepository schemaRepository; + @Mock + private GameAchievementSchemaMetaRepository schemaMetaRepository; + @Mock + private AchievementSyncService achievementSyncService; + @Mock + private SteamApiService steamApiService; + @Mock + private SteamCdnService steamCdnService; + + private ImageRepairService service; + + @BeforeEach + void setUp() { + service = new ImageRepairService(brokenImageRepository, userGameRepository, steamGameRepository, schemaRepository, + schemaMetaRepository, achievementSyncService, steamApiService, steamCdnService, BASE, 200, 3, 30); + } + + private void due(BrokenImage... rows) { + when(brokenImageRepository.findDue(eq(3), any(), eq(200))).thenReturn(List.of(rows)); + } + + private static BrokenImage row(long appid, ImageKind kind, Instant reportedAt) { + return new BrokenImage(appid, kind, reportedAt, null, 0); + } + + private static GameAchievementSchema icon(long appid, String apiName, String url) { + return new GameAchievementSchema(appid, apiName, apiName, null, url, url + "_gray", false); + } + + @Test + void report_StoresAReportForAnOwnedSteamGame() { + when(userGameRepository.findActiveAppids(1L, List.of(620L))).thenReturn(List.of(620L)); + + service.report(1L, 620L, ImageKind.COVER); + + verify(brokenImageRepository).report(eq(620L), eq("COVER"), any()); + } + + @Test + void report_DropsGamesTheUserDoesNotOwn() { + when(userGameRepository.findActiveAppids(1L, List.of(620L))).thenReturn(List.of()); + + service.report(1L, 620L, ImageKind.COVER); + + verify(brokenImageRepository, never()).report(anyLong(), anyString(), any()); + } + + @Test + void report_DropsGamesOffSteam() { + service.report(1L, -1234L, ImageKind.COVER); + + verifyNoInteractions(userGameRepository, brokenImageRepository); + } + + @Test + void repairNightly_DeletesAFalseCoverReportWithoutAskingTheStore() { + due(row(620L, ImageKind.COVER, Instant.now())); + when(steamCdnService.isAvailable(FIXED_620)).thenReturn(true); + + service.repairNightly(); + + verify(brokenImageRepository).deleteReportedUpTo(eq(620L), eq("COVER"), any()); + verifyNoInteractions(steamApiService); + verify(steamGameRepository, never()).setImageUrl(anyLong(), anyString()); + } + + @Test + void repairNightly_StoresTheHashedStoreHeaderWhenTheFixedOneIsGone() { + due(row(620L, ImageKind.COVER, Instant.now())); + when(steamCdnService.isAvailable(FIXED_620)).thenReturn(false); + when(steamApiService.getStoreHeaderUrls(List.of(620L))).thenReturn(Map.of(620L, HASHED_620)); + when(steamCdnService.isAvailable(HASHED_620)).thenReturn(true); + + service.repairNightly(); + + verify(steamGameRepository).setImageUrl(620L, HASHED_620); + verify(brokenImageRepository).deleteReportedUpTo(eq(620L), eq("COVER"), any()); + } + + @Test + void repairNightly_ChecksTheStoredRepairFirst() { + due(row(620L, ImageKind.COVER, Instant.now())); + SteamGame game = new SteamGame(620L, "Portal 2", null, Instant.now()); + game.setImageUrl(HASHED_620); + when(steamGameRepository.findAllById(List.of(620L))).thenReturn(List.of(game)); + when(steamCdnService.isAvailable(HASHED_620)).thenReturn(true); + + service.repairNightly(); + + verify(steamCdnService, never()).isAvailable(FIXED_620); + verify(brokenImageRepository).deleteReportedUpTo(eq(620L), eq("COVER"), any()); + } + + @Test + void repairNightly_CountsAnAttemptWhenTheStoreHasNoHeader() { + due(row(620L, ImageKind.COVER, Instant.now())); + when(steamApiService.getStoreHeaderUrls(List.of(620L))).thenReturn(Map.of()); + + service.repairNightly(); + + verify(brokenImageRepository).markChecked(eq(620L), eq("COVER"), any(), eq(3)); + verify(brokenImageRepository, never()).deleteReportedUpTo(anyLong(), anyString(), any()); + verify(steamGameRepository, never()).setImageUrl(anyLong(), anyString()); + } + + @Test + void repairNightly_CountsAnAttemptWhenTheStoreHeaderIsBrokenToo() { + due(row(620L, ImageKind.COVER, Instant.now())); + when(steamApiService.getStoreHeaderUrls(List.of(620L))).thenReturn(Map.of(620L, HASHED_620)); + + service.repairNightly(); + + verify(brokenImageRepository).markChecked(eq(620L), eq("COVER"), any(), eq(3)); + verify(steamGameRepository, never()).setImageUrl(anyLong(), anyString()); + } + + @Test + void repairNightly_DeletesTheIconReportWhenTheRefreshBringsNewUrls() throws Exception { + Instant reported = Instant.now().minus(Duration.ofHours(2)); + due(row(620L, ImageKind.ACHIEVEMENT_ICONS, reported)); + when(schemaMetaRepository.findById(620L)).thenReturn(Optional.of( + new GameAchievementSchemaMeta(620L, 1, true, reported.minus(Duration.ofDays(3))))); + when(schemaRepository.findByAppid(620L)) + .thenReturn(List.of(icon(620L, "A", "old"))) + .thenReturn(List.of(icon(620L, "A", "new"))); + + service.repairNightly(); + + verify(achievementSyncService).refreshSchemaNow(620L); + verify(brokenImageRepository).deleteReportedUpTo(eq(620L), eq("ACHIEVEMENT_ICONS"), any()); + } + + @Test + void repairNightly_CountsAnAttemptWhenTheIconsStayTheSame() throws Exception { + due(row(620L, ImageKind.ACHIEVEMENT_ICONS, Instant.now())); + when(schemaRepository.findByAppid(620L)).thenReturn(List.of(icon(620L, "A", "same"))); + + service.repairNightly(); + + verify(achievementSyncService).refreshSchemaNow(620L); + verify(brokenImageRepository).markChecked(eq(620L), eq("ACHIEVEMENT_ICONS"), any(), eq(3)); + verify(brokenImageRepository, never()).deleteReportedUpTo(anyLong(), anyString(), any()); + } + + @Test + void repairNightly_DoesNotAskSteamWhenTheSchemaIsNewerThanTheReport() throws Exception { + Instant reported = Instant.now().minus(Duration.ofHours(5)); + due(row(620L, ImageKind.ACHIEVEMENT_ICONS, reported)); + when(schemaMetaRepository.findById(620L)).thenReturn(Optional.of( + new GameAchievementSchemaMeta(620L, 1, true, reported.plus(Duration.ofHours(1))))); + + service.repairNightly(); + + verify(achievementSyncService, never()).refreshSchemaNow(anyLong()); + verify(brokenImageRepository).deleteReportedUpTo(eq(620L), eq("ACHIEVEMENT_ICONS"), any()); + } + + @Test + void repairNightly_ASteamFailureCountsAnAttemptAndTheNextRowStillRuns() throws Exception { + due(row(620L, ImageKind.ACHIEVEMENT_ICONS, Instant.now()), row(220L, ImageKind.ACHIEVEMENT_ICONS, Instant.now())); + doThrow(new IllegalStateException("Steam 500")).when(achievementSyncService).refreshSchemaNow(620L); + when(schemaRepository.findByAppid(620L)).thenReturn(List.of(icon(620L, "A", "same"))); + when(schemaRepository.findByAppid(220L)) + .thenReturn(List.of(icon(220L, "A", "old"))) + .thenReturn(List.of(icon(220L, "A", "new"))); + + service.repairNightly(); + + verify(brokenImageRepository).markChecked(eq(620L), eq("ACHIEVEMENT_ICONS"), any(), anyInt()); + verify(brokenImageRepository).deleteReportedUpTo(eq(220L), eq("ACHIEVEMENT_ICONS"), any()); + } + + @Test + void repairNightly_DeletesAFalseIconReportWithoutAskingSteam() throws Exception { + due(row(620L, ImageKind.ACHIEVEMENT_ICONS, Instant.now())); + when(schemaRepository.findByAppid(620L)).thenReturn(List.of(icon(620L, "A", "https://x/a.jpg"))); + when(steamCdnService.isAvailable(anyString())).thenReturn(true); + + service.repairNightly(); + + verify(achievementSyncService, never()).refreshSchemaNow(anyLong()); + verify(brokenImageRepository).deleteReportedUpTo(eq(620L), eq("ACHIEVEMENT_ICONS"), any()); + } + + @Test + void repairNightly_ASchemaWithoutIconsIsNoFalseReport() throws Exception { + due(row(620L, ImageKind.ACHIEVEMENT_ICONS, Instant.now())); + when(schemaRepository.findByAppid(620L)).thenReturn(List.of()); + + service.repairNightly(); + + verify(achievementSyncService).refreshSchemaNow(620L); + verifyNoInteractions(steamCdnService); + } + + @Test + void repairNightly_AFailedCoverWriteCountsAnAttempt() { + due(row(620L, ImageKind.COVER, Instant.now())); + when(steamApiService.getStoreHeaderUrls(List.of(620L))).thenReturn(Map.of(620L, HASHED_620)); + when(steamCdnService.isAvailable(HASHED_620)).thenReturn(true); + doThrow(new IllegalStateException("db down")).when(steamGameRepository).setImageUrl(620L, HASHED_620); + + service.repairNightly(); + + verify(brokenImageRepository).markChecked(eq(620L), eq("COVER"), any(), eq(3)); + } + + @Test + void repairNightly_MarksTheRunStartSoAReportDuringTheRunStaysDue() { + due(row(620L, ImageKind.COVER, Instant.now())); + when(steamApiService.getStoreHeaderUrls(List.of(620L))).thenReturn(Map.of()); + Instant before = Instant.now(); + + service.repairNightly(); + + org.mockito.ArgumentCaptor checkedAt = org.mockito.ArgumentCaptor.forClass(Instant.class); + verify(brokenImageRepository).markChecked(eq(620L), eq("COVER"), checkedAt.capture(), eq(3)); + verify(brokenImageRepository).findDue(eq(3), eq(checkedAt.getValue().minus(Duration.ofDays(30))), eq(200)); + org.assertj.core.api.Assertions.assertThat(checkedAt.getValue()).isAfterOrEqualTo(before); + } + + @Test + void repairNightly_StopsWhenInterrupted() throws Exception { + due(row(620L, ImageKind.ACHIEVEMENT_ICONS, Instant.now()), row(220L, ImageKind.ACHIEVEMENT_ICONS, Instant.now())); + doAnswer(call -> { throw new InterruptedException(); }).when(achievementSyncService).refreshSchemaNow(620L); + + service.repairNightly(); + + verify(achievementSyncService, never()).refreshSchemaNow(220L); + // Clear the flag for the next test on this thread. + org.assertj.core.api.Assertions.assertThat(Thread.interrupted()).isTrue(); + } +} diff --git a/src/test/java/core_service/modules/steam_adapter/service/SteamApiServiceTest.java b/src/test/java/core_service/modules/steam_adapter/service/SteamApiServiceTest.java index f13e7dd..41c9f04 100644 --- a/src/test/java/core_service/modules/steam_adapter/service/SteamApiServiceTest.java +++ b/src/test/java/core_service/modules/steam_adapter/service/SteamApiServiceTest.java @@ -6,6 +6,8 @@ import core_service.modules.steam_adapter.api.dto.response.SteamPlayerSummaryRes import core_service.modules.steam_adapter.api.dto.response.SteamProfileAndGamesResponse; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.test.util.ReflectionTestUtils; @@ -282,6 +284,34 @@ class SteamApiServiceTest { server.verify(); } + @Test + void getStoreHeaderUrls_BuildsTheHashedUrlAndSkipsAppsWithoutAssets() { + server.expect(requestTo(containsString("/IStoreBrowseService/GetItems/v1/"))) + .andExpect(queryParam("input_json", containsString("%22data_request%22%3A%7B%22include_assets%22%3Atrue%7D"))) + .andRespond(withSuccess(""" + {"response": {"store_items": [ + {"appid": 620, "success": 1, + "assets": {"asset_url_format": "steam/apps/620/${FILENAME}?t=1776125736", + "header": "7983574d464e6559ac7e24275727f73a8bcca1f3/header.jpg"}}, + {"appid": 3012600, "success": 1}, + {"appid": 42, "success": 1, "assets": {"asset_url_format": "steam/apps/42/${FILENAME}"}} + ]}} + """, MediaType.APPLICATION_JSON)); + + Map urls = steamApiService.getStoreHeaderUrls(List.of(620L, 3012600L, 42L)); + + assertThat(urls).containsOnly(Map.entry(620L, + "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/620/7983574d464e6559ac7e24275727f73a8bcca1f3/header.jpg?t=1776125736")); + } + + @Test + void getStoreHeaderUrls_ServerError_IsEmpty() { + server.expect(requestTo(containsString("/IStoreBrowseService/GetItems/v1/"))) + .andRespond(withServerError()); + + assertThat(steamApiService.getStoreHeaderUrls(List.of(620L))).isEmpty(); + } + @Test void getPlayerAchievements_Forbidden_IsEmpty() { server.expect(requestTo(containsString("/ISteamUserStats/GetPlayerAchievements/v1/"))) @@ -327,6 +357,15 @@ class SteamApiServiceTest { assertThat(stats.get().achievements()).isEmpty(); } + @ParameterizedTest + @ValueSource(ints = {408, 429}) + void getSchemaForGame_TooManyRequestsOrTimeout_Throws(int status) { + server.expect(requestTo(containsString("/ISteamUserStats/GetSchemaForGame/v2/"))) + .andRespond(withStatus(HttpStatus.valueOf(status))); + + assertThatThrownBy(() -> steamApiService.getSchemaForGame(620)).isInstanceOf(RestClientException.class); + } + @Test void getSchemaForGame_ClientError_IsEmpty() { server.expect(requestTo(containsString("/ISteamUserStats/GetSchemaForGame/v2/"))) diff --git a/src/test/java/core_service/modules/steam_adapter/service/SteamCdnServiceTest.java b/src/test/java/core_service/modules/steam_adapter/service/SteamCdnServiceTest.java new file mode 100644 index 0000000..4618453 --- /dev/null +++ b/src/test/java/core_service/modules/steam_adapter/service/SteamCdnServiceTest.java @@ -0,0 +1,62 @@ +package core_service.modules.steam_adapter.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestClient; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withException; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +class SteamCdnServiceTest { + + private static final String URL = "https://cdn.cloudflare.steamstatic.com/steam/apps/620/header.jpg"; + + private SteamCdnService steamCdnService; + private MockRestServiceServer server; + + @BeforeEach + void setUp() { + RestClient.Builder builder = RestClient.builder(); + server = MockRestServiceServer.bindTo(builder).build(); + steamCdnService = new SteamCdnService(builder.build()); + } + + @Test + void isAvailable_OkIsTrue() { + server.expect(requestTo(URL)).andExpect(method(HttpMethod.HEAD)).andRespond(withSuccess()); + + assertThat(steamCdnService.isAvailable(URL)).isTrue(); + } + + @Test + void isAvailable_NotFoundIsFalse() { + server.expect(requestTo(URL)).andRespond(withStatus(HttpStatus.NOT_FOUND)); + + assertThat(steamCdnService.isAvailable(URL)).isFalse(); + } + + @Test + void isAvailable_RedirectIsFalse() { + server.expect(requestTo(URL)).andRespond(withStatus(HttpStatus.MOVED_PERMANENTLY) + .header(HttpHeaders.LOCATION, "https://example.com/")); + + assertThat(steamCdnService.isAvailable(URL)).isFalse(); + } + + @Test + void isAvailable_NetworkErrorIsFalse() { + server.expect(requestTo(URL)).andRespond(withException(new IOException("timeout"))); + + assertThat(steamCdnService.isAvailable(URL)).isFalse(); + } +} diff --git a/src/test/java/core_service/shared/test/AbstractIntegrationTest.java b/src/test/java/core_service/shared/test/AbstractIntegrationTest.java index 56121a5..ac366ac 100644 --- a/src/test/java/core_service/shared/test/AbstractIntegrationTest.java +++ b/src/test/java/core_service/shared/test/AbstractIntegrationTest.java @@ -20,7 +20,7 @@ import java.util.Base64; @Tag("IntegrationTest") // The nightly sync would call Steam if a test ran at 23:55; speedrun.com needs no key, so it is switched off. @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = {"app.library.nightly-sync-cron=-", "app.speedrun.enabled=false"}) + properties = {"app.library.nightly-sync-cron=-", "app.library.image-repair-cron=-", "app.speedrun.enabled=false"}) @ActiveProfiles("dev") public abstract class AbstractIntegrationTest {