diff --git a/readme.md b/readme.md index 6407dad..5656125 100644 --- a/readme.md +++ b/readme.md @@ -103,6 +103,11 @@ method calling another service's method that needs more. | `PUT` | `/api/library/games/{appid}/status` (status only, keeps rating, notes, priority) | JWT | | `PUT` | `/api/library/games/{appid}/finished-hint` (`{dismissed: ENDING\|PLAYTIME}`, "Not yet"; `204`) | JWT | | `PUT` | `/api/library/games/{appid}/hidden` (`{hidden}`; `409 MANUAL_GAME` for a game added by hand) | JWT | +| `POST` | `/api/library/imports` (`{source: TEXT\|CSV\|SEARCH, rows: [{title, platform, playtimeMinutes, status, igdbId}]}`, 1–1000 rows; `202 {id}` in `QUEUED`, `409 IMPORT_RUNNING`, `503 IGDB_OFF`) | JWT | +| `GET` | `/api/library/imports` (`{imports}`, the last 20), `/api/library/imports/{id}` (status, progress counts, `ahead` while it waits in a queue, `attempts`; the rows with their candidates while `READY`) | JWT | +| `POST` | `/api/library/imports/{id}/save` (`{rows: [{lineNo, igdbId, platform, status, playtimeMinutes}]}`; `202`, then poll until `SAVED`; `409 IMPORT_NOT_READY`) | JWT | +| `POST` | `/api/library/imports/{id}/retry` (a `FAILED` import; `202`, `409 IMPORT_NOT_FAILED` / `IMPORT_RUNNING`) | JWT | +| `GET`, `DELETE` | `/api/library/imports/{id}/undo-check` (`{games, changed}`), `/api/library/imports/{id}/games` (undo; `204`, `409 IMPORT_NOT_SAVED`) | JWT | | `POST` | `/api/library/sync` | JWT | | `POST` | `/api/library/broken-images` (`{appid, kind}`, kind `COVER` or `ACHIEVEMENT_ICONS`; `204`) | JWT | | `GET` | `/api/library/picker?maxMinutes=&mood=&count=&exclude=` | JWT | @@ -163,7 +168,7 @@ flowchart LR | Module | Content | |---|---| -| `library` | Sync, backlog, manual games, playtime history, wishlist, achievements, public profile data | +| `library` | Sync, backlog, manual games, imports, playtime history, wishlist, achievements, public profile data | | `stats` | Public numbers for the landing page | | `version` | `GET /api/version`: commit and build time of the running image | | `account` | Deleted accounts, driven by Auth: tombstones in `deleted_user`, purge and orphan prune of the per-user rows | @@ -265,6 +270,46 @@ id for negative keys. Search and add check both keys of a game. A game added und a second row when IGDB learns its Steam appid later and the user owns it on Steam; an auditor then merges the two (see "Merging duplicates"). +### Imports + +Many games at once (V22, `GameImportService`): a list, a CSV or several picks of the game search become one +`game_import` with a `game_import_row` per line. `game_import` is also the queue of `GameImportWorker`, which has +two queues, each on one thread for all users: + +- **matching**: IGDB searches with typo help. One thread, so an import never takes the whole IGDB budget from the + interactive search; other users' searches may wait for the limiter meanwhile. Up to 5 candidates per line are + stored whole (JSONB), so the save needs no IGDB call, and `exact` marks a best one with the line's name. +- **saving**: adds the chosen lines the way the manual add does (same keys and duplicate check) with + `user_game.import_id` in the same transaction, then loads their time to beat and moods. A save never waits + behind another user's big matching. + +Each line is stored when done, so a stopped try goes on where it stopped. A worker sets `progress_at` when it takes +an import and after every line. A watchdog (every minute) stops a try without progress for 5 minutes +(`app.library.import-stall-minutes`): the import goes to the end of its queue and the queue gets a new thread. A +restart counts as a stopped try too. After 3 tries the import is `FAILED`; the user can try it again. One import +per user can be in the matching queue (a unique partial index). + +```mermaid +stateDiagram-v2 + [*] --> QUEUED: POST /imports + QUEUED --> MATCHING: matching worker takes it + MATCHING --> READY: every line matched + READY --> SAVING: POST /save (the review) + SAVING --> SAVED: games added, time to beat and moods loaded + SAVED --> UNDONE: DELETE /games (undo) + MATCHING --> QUEUED: no progress for 5 min or restart + SAVING --> SAVING: no progress for 5 min or restart (end of the queue) + MATCHING --> FAILED: 3rd stopped try + SAVING --> FAILED: 3rd stopped try + FAILED --> QUEUED: POST /retry (stopped while matching) + FAILED --> SAVING: POST /retry (stopped while saving) + FAILED --> UNDONE: DELETE /games (the games added before it failed) +``` + +The undo deletes the games of the import that are still `MANUAL` (a game the Steam sync took over stays). A removed +row the import brings back gets no `import_id`: it holds rating, notes and history from before, so the undo keeps it; +`undo-check` counts those with a rating, notes, priority or a status change after the save. + ## Run locally ### Local development without keys (profile `local`) diff --git a/src/main/java/core_service/modules/account/repo/AccountCleanupRepository.java b/src/main/java/core_service/modules/account/repo/AccountCleanupRepository.java index fd2b1eb..5df0ed8 100644 --- a/src/main/java/core_service/modules/account/repo/AccountCleanupRepository.java +++ b/src/main/java/core_service/modules/account/repo/AccountCleanupRepository.java @@ -21,11 +21,11 @@ public class AccountCleanupRepository { // Children before user_game (user_game_status_change would also go by its cascade), the tombstones last. private static final List TABLES = List.of("user_game_status_change", "user_game_achievement", "user_game_achievement_meta", "user_game_playtime_day", "wishlist_game", "backlog_snapshot", - "user_steam_sync", "user_game", "deleted_user"); + "user_steam_sync", "user_game", "game_import", "deleted_user"); // Every user with data has a row in one of these; deleteRows then removes the child rows too. private static final List ORPHAN_SOURCES = List.of("user_game", "user_steam_sync", "wishlist_game", - "deleted_user"); + "game_import", "deleted_user"); private final NamedParameterJdbcTemplate jdbc; diff --git a/src/main/java/core_service/modules/library/api/GameImportController.java b/src/main/java/core_service/modules/library/api/GameImportController.java new file mode 100644 index 0000000..85f0fca --- /dev/null +++ b/src/main/java/core_service/modules/library/api/GameImportController.java @@ -0,0 +1,99 @@ +package core_service.modules.library.api; + +import core_service.modules.library.api.dto.ImportHistoryDto; +import core_service.modules.library.api.dto.ImportRequest; +import core_service.modules.library.api.dto.ImportSaveRequest; +import core_service.modules.library.service.GameImportService; +import core_service.shared.security.UserPrincipal; +import jakarta.validation.Valid; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** + * Imports of many games at once, see GameImport. + */ +@RestController +@RequestMapping("/api/library/imports") +@RequiredArgsConstructor +public class GameImportController { + + private static final Map NOT_FOUND = Map.of("code", "IMPORT_NOT_FOUND", "message", "No such import"); + private static final ResponseEntity IMPORT_RUNNING = ResponseEntity.status(HttpStatus.CONFLICT) + .body(Map.of("code", "IMPORT_RUNNING", "message", "Another import is still being matched")); + + private final GameImportService importService; + + @PostMapping + public ResponseEntity create(@AuthenticationPrincipal UserPrincipal principal, + @Valid @RequestBody ImportRequest request) { + return switch (importService.create(principal.getIdAsLong(), request.source(), request.rows())) { + case GameImportService.CreateResult.Created created -> ResponseEntity.status(HttpStatus.ACCEPTED) + .body(Map.of("id", created.id())); + case GameImportService.CreateResult.AlreadyMatching ignored -> IMPORT_RUNNING; + case GameImportService.CreateResult.SearchUnavailable ignored -> ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE) + .body(Map.of("code", "IGDB_OFF", "message", "Importing games needs the IGDB app")); + }; + } + + @GetMapping + public ImportHistoryDto list(@AuthenticationPrincipal UserPrincipal principal) { + return new ImportHistoryDto(importService.list(principal.getIdAsLong())); + } + + @GetMapping("/{id}") + public ResponseEntity get(@AuthenticationPrincipal UserPrincipal principal, @PathVariable("id") long id) { + return importService.get(principal.getIdAsLong(), id) + .>map(ResponseEntity::ok) + .orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body(NOT_FOUND)); + } + + @PostMapping("/{id}/save") + public ResponseEntity save(@AuthenticationPrincipal UserPrincipal principal, @PathVariable("id") long id, + @Valid @RequestBody ImportSaveRequest request) { + return switch (importService.save(principal.getIdAsLong(), id, request.rows())) { + case DONE -> ResponseEntity.accepted().build(); + case NOT_FOUND -> ResponseEntity.status(HttpStatus.NOT_FOUND).body(NOT_FOUND); + case WRONG_STATUS, ALREADY_MATCHING -> ResponseEntity.status(HttpStatus.CONFLICT) + .body(Map.of("code", "IMPORT_NOT_READY", "message", "The import is still matching or saved already")); + }; + } + + @PostMapping("/{id}/retry") + public ResponseEntity retry(@AuthenticationPrincipal UserPrincipal principal, @PathVariable("id") long id) { + return switch (importService.retry(principal.getIdAsLong(), id)) { + case DONE -> ResponseEntity.accepted().build(); + case NOT_FOUND -> ResponseEntity.status(HttpStatus.NOT_FOUND).body(NOT_FOUND); + case WRONG_STATUS -> ResponseEntity.status(HttpStatus.CONFLICT) + .body(Map.of("code", "IMPORT_NOT_FAILED", "message", "Only a failed import can be tried again")); + case ALREADY_MATCHING -> IMPORT_RUNNING; + }; + } + + @GetMapping("/{id}/undo-check") + public ResponseEntity undoCheck(@AuthenticationPrincipal UserPrincipal principal, @PathVariable("id") long id) { + return importService.undoCheck(principal.getIdAsLong(), id) + .>map(ResponseEntity::ok) + .orElseGet(() -> ResponseEntity.status(HttpStatus.NOT_FOUND).body(NOT_FOUND)); + } + + @DeleteMapping("/{id}/games") + public ResponseEntity undo(@AuthenticationPrincipal UserPrincipal principal, @PathVariable("id") long id) { + return switch (importService.undo(principal.getIdAsLong(), id)) { + case DONE -> ResponseEntity.noContent().build(); + case NOT_FOUND -> ResponseEntity.status(HttpStatus.NOT_FOUND).body(NOT_FOUND); + case WRONG_STATUS, ALREADY_MATCHING -> ResponseEntity.status(HttpStatus.CONFLICT) + .body(Map.of("code", "IMPORT_NOT_SAVED", "message", "Only a saved import can be undone")); + }; + } +} diff --git a/src/main/java/core_service/modules/library/api/ManualGameController.java b/src/main/java/core_service/modules/library/api/ManualGameController.java index fe0d2ec..25eb42c 100644 --- a/src/main/java/core_service/modules/library/api/ManualGameController.java +++ b/src/main/java/core_service/modules/library/api/ManualGameController.java @@ -52,7 +52,7 @@ public class ManualGameController { public ResponseEntity add(@AuthenticationPrincipal UserPrincipal principal, @Valid @RequestBody AddGameRequest request) { int minutes = request.playtimeMinutes() != null ? request.playtimeMinutes() : 0; - return switch (manualGameService.add(principal.getIdAsLong(), request.igdbId(), request.platform().trim(), + return switch (manualGameService.add(principal.getIdAsLong(), request.igdbId(), request.platform(), request.status(), minutes)) { case ManualGameService.AddResult.Added added -> ResponseEntity .created(URI.create("/api/library/games/" + added.appid())) diff --git a/src/main/java/core_service/modules/library/api/dto/GameImportDto.java b/src/main/java/core_service/modules/library/api/dto/GameImportDto.java new file mode 100644 index 0000000..cf16573 --- /dev/null +++ b/src/main/java/core_service/modules/library/api/dto/GameImportDto.java @@ -0,0 +1,32 @@ +package core_service.modules.library.api.dto; + +import core_service.modules.library.domain.GameImport; +import core_service.modules.library.domain.GameStatus; + +import java.time.Instant; +import java.util.List; + +/** + * An import and its progress, see {@link GameImport}. + * + * @param ahead while it waits in a queue (QUEUED, or SAVING before its save started): the imports of any user before + * it; else null + * @param attempts tries a watchdog or a restart stopped; FAILED after 3 + * @param rows only for an import in READY that was asked for by its id; else null + */ +public record GameImportDto(long id, GameImport.Source source, GameImport.Status status, int total, int matched, + int chosen, int saved, int added, Integer ahead, GameImport.Step step, int attempts, + Instant createdAt, Instant savedAt, List rows) { + + /** + * @param exact the first candidate has the line's name; see the review's preselection + */ + public record Row(int lineNo, String title, String platform, Integer playtimeMinutes, GameStatus status, + boolean searchFailed, boolean exact, List candidates) {} + + /** + * @param inLibrary the user has this game already, from Steam or by hand + */ + public record Candidate(long igdbId, String name, Integer year, String imageUrl, List platforms, + boolean inLibrary) {} +} diff --git a/src/main/java/core_service/modules/library/api/dto/ImportHistoryDto.java b/src/main/java/core_service/modules/library/api/dto/ImportHistoryDto.java new file mode 100644 index 0000000..1159f84 --- /dev/null +++ b/src/main/java/core_service/modules/library/api/dto/ImportHistoryDto.java @@ -0,0 +1,8 @@ +package core_service.modules.library.api.dto; + +import java.util.List; + +/** + * @param imports the user's last imports, newest first + */ +public record ImportHistoryDto(List imports) {} diff --git a/src/main/java/core_service/modules/library/api/dto/ImportRequest.java b/src/main/java/core_service/modules/library/api/dto/ImportRequest.java new file mode 100644 index 0000000..3c2a948 --- /dev/null +++ b/src/main/java/core_service/modules/library/api/dto/ImportRequest.java @@ -0,0 +1,38 @@ +package core_service.modules.library.api.dto; + +import core_service.modules.library.domain.GameImport; +import core_service.modules.library.domain.GameStatus; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; + +import java.util.List; + +public record ImportRequest( + @NotNull GameImport.Source source, + @NotEmpty @Size(max = MAX_ROWS) List<@NotNull @Valid Row> rows +) { + + public static final int MAX_ROWS = 1000; + + /** + * @param igdbId the game already picked in the search; then the title is not searched + */ + public record Row( + @Size(max = 100) String title, + @Size(max = ManualGameUpdate.MAX_PLATFORM_LENGTH) String platform, + @Min(0) @Max(ManualGameUpdate.MAX_MINUTES) Integer playtimeMinutes, + GameStatus status, + @Min(1) Long igdbId + ) { + + @AssertTrue(message = "title or igdbId is needed") + public boolean isTitleOrIgdbId() { + return igdbId != null || (title != null && !title.isBlank()); + } + } +} diff --git a/src/main/java/core_service/modules/library/api/dto/ImportSaveRequest.java b/src/main/java/core_service/modules/library/api/dto/ImportSaveRequest.java new file mode 100644 index 0000000..4645b81 --- /dev/null +++ b/src/main/java/core_service/modules/library/api/dto/ImportSaveRequest.java @@ -0,0 +1,28 @@ +package core_service.modules.library.api.dto; + +import core_service.modules.library.domain.GameStatus; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Size; + +import java.util.List; + +/** + * The lines the user checked in the review, each with the game picked. + */ +public record ImportSaveRequest(@NotNull @Size(max = ImportRequest.MAX_ROWS) List<@NotNull @Valid Row> rows) { + + /** + * @param igdbId a candidate of the line, or a game from a new search + */ + public record Row( + @Min(1) int lineNo, + @NotNull @Min(1) Long igdbId, + @NotBlank @Size(max = ManualGameUpdate.MAX_PLATFORM_LENGTH) String platform, + @NotNull GameStatus status, + @Min(0) @Max(ManualGameUpdate.MAX_MINUTES) Integer playtimeMinutes + ) {} +} diff --git a/src/main/java/core_service/modules/library/api/dto/UndoCheckDto.java b/src/main/java/core_service/modules/library/api/dto/UndoCheckDto.java new file mode 100644 index 0000000..73b80f7 --- /dev/null +++ b/src/main/java/core_service/modules/library/api/dto/UndoCheckDto.java @@ -0,0 +1,7 @@ +package core_service.modules.library.api.dto; + +/** + * @param games the import's games an undo would remove + * @param changed of these, the games with a rating, notes, priority or a new status since the save + */ +public record UndoCheckDto(int games, int changed) {} diff --git a/src/main/java/core_service/modules/library/domain/GameImport.java b/src/main/java/core_service/modules/library/domain/GameImport.java new file mode 100644 index 0000000..8006f70 --- /dev/null +++ b/src/main/java/core_service/modules/library/domain/GameImport.java @@ -0,0 +1,55 @@ +package core_service.modules.library.domain; + +import core_service.modules.igdb_adapter.service.IgdbApiService; + +import java.time.Instant; +import java.util.List; + +/** + * Many games at once (V22): the lines of a list or a CSV, matched with IGDB in the background, reviewed by the user + * and then saved in the background as games added by hand. The games keep the import's id, so the import can be + * undone. + * + * @param matched lines matched so far, of total + * @param chosen lines the user checked in the review + * @param saved of chosen, the lines the save has done + * @param added of saved, the lines that became a game in the library + * @param running a worker runs the import now; false while it waits in its queue + */ +public record GameImport(long id, long userId, Source source, Status status, int total, int matched, int chosen, + int saved, int added, boolean running, int attempts, Step step, Instant createdAt, + Instant queuedAt, Instant savedAt) { + + public enum Source { TEXT, CSV, SEARCH } + + public enum Status { QUEUED, MATCHING, READY, SAVING, SAVED, UNDONE, FAILED } + + /** + * Of a save: null while the games are added, then DETAILS while their time to beat and moods load. + */ + public enum Step { DETAILS } + + public enum SaveResult { ADDED, SKIPPED } + + /** + * One input line; every field but lineNo may be missing. + * + * @param igdbId the game the user already picked in the search (source SEARCH); then title is not searched + */ + public record Line(int lineNo, String title, String platform, Integer playtimeMinutes, GameStatus status, + Long igdbId) {} + + /** + * A line after the matching. + * + * @param searchFailed IGDB did not answer for this line + * @param exact the first candidate has the line's name (case and punctuation ignored) + * @param candidates best first, at most five + */ + public record Row(Line line, boolean searchFailed, boolean exact, List candidates) {} + + /** + * The user's pick for a line in the review. + */ + public record Choice(int lineNo, long igdbId, String platform, GameStatus status, int playtimeMinutes) {} +} diff --git a/src/main/java/core_service/modules/library/domain/UserGame.java b/src/main/java/core_service/modules/library/domain/UserGame.java index 550f1ba..0df5fae 100644 --- a/src/main/java/core_service/modules/library/domain/UserGame.java +++ b/src/main/java/core_service/modules/library/domain/UserGame.java @@ -87,6 +87,10 @@ public class UserGame { @Column(name = "platform", length = 32) private String platform; + /** The import that added this game (V22), so the import can be undone. Null for any other add. */ + @Column(name = "import_id") + private Long importId; + public UserGame(Long userId, Long appid, int playtimeMinutes, Instant firstSeenAt, Instant lastSeenAt) { this.userId = userId; this.appid = appid; diff --git a/src/main/java/core_service/modules/library/repo/GameImportRepository.java b/src/main/java/core_service/modules/library/repo/GameImportRepository.java new file mode 100644 index 0000000..ed4f37d --- /dev/null +++ b/src/main/java/core_service/modules/library/repo/GameImportRepository.java @@ -0,0 +1,283 @@ +package core_service.modules.library.repo; + +import core_service.modules.igdb_adapter.service.IgdbApiService; +import core_service.modules.library.domain.GameImport; +import core_service.modules.library.domain.GameStatus; +import lombok.RequiredArgsConstructor; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; +import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.stereotype.Repository; +import tools.jackson.databind.json.JsonMapper; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Imports and their lines (V22), the queues of the import workers, and the games an import added. + */ +@Repository +@RequiredArgsConstructor +public class GameImportRepository { + + private static final String SELECT_IMPORT = """ + select i.*, r.* from game_import i cross join lateral ( + select count(*) filter (where matched_at is not null) as matched, + count(*) filter (where chosen_igdb_id is not null) as chosen, + count(*) filter (where save_result is not null) as saved, + count(*) filter (where save_result = 'ADDED') as added + from game_import_row where import_id = i.id) r + """; + + private static final RowMapper IMPORT = (rs, row) -> new GameImport(rs.getLong("id"), + rs.getLong("user_id"), GameImport.Source.valueOf(rs.getString("source")), + GameImport.Status.valueOf(rs.getString("status")), rs.getInt("total"), rs.getInt("matched"), + rs.getInt("chosen"), rs.getInt("saved"), rs.getInt("added"), rs.getTimestamp("progress_at") != null, + rs.getInt("attempts"), rs.getString("step") != null ? GameImport.Step.valueOf(rs.getString("step")) : null, + rs.getTimestamp("created_at").toInstant(), rs.getTimestamp("queued_at").toInstant(), + instant(rs.getTimestamp("saved_at"))); + + private final NamedParameterJdbcTemplate jdbc; + private final JsonMapper jsonMapper; + + /** + * Puts the import in the matching queue. The V22 index refuses a second import there per user with a + * DuplicateKeyException. + */ + public long create(long userId, GameImport.Source source, List lines) { + GeneratedKeyHolder key = new GeneratedKeyHolder(); + jdbc.update("insert into game_import (user_id, source, status, total) values (:user, :source, 'QUEUED', :total)", + new MapSqlParameterSource("user", userId).addValue("source", source.name()).addValue("total", lines.size()), + key, new String[]{"id"}); + long id = key.getKeyAs(Long.class); + jdbc.batchUpdate("insert into game_import_row (import_id, line_no, title, platform, playtime_minutes, status, " + + "igdb_id) values (:id, :line, :title, :platform, :minutes, :status, :igdb)", + lines.stream().map(line -> new MapSqlParameterSource("id", id).addValue("line", line.lineNo()) + .addValue("title", line.title()).addValue("platform", line.platform()) + .addValue("minutes", line.playtimeMinutes()) + .addValue("status", line.status() != null ? line.status().name() : null) + .addValue("igdb", line.igdbId())).toArray(MapSqlParameterSource[]::new)); + return id; + } + + public Optional find(long userId, long id) { + return jdbc.query(SELECT_IMPORT + "where i.id = :id and i.user_id = :user", + new MapSqlParameterSource("id", id).addValue("user", userId), IMPORT).stream().findFirst(); + } + + public List findByUser(long userId, int limit) { + return jdbc.query(SELECT_IMPORT + "where i.user_id = :user order by i.id desc limit :limit", + new MapSqlParameterSource("user", userId).addValue("limit", limit), IMPORT); + } + + /** + * The imports before this one in its queue (of any user). + */ + public int countAhead(GameImport waiting, Collection queue) { + return jdbc.queryForObject("select count(*) from game_import where status in (:queue) " + + "and (queued_at, id) < (:queuedAt, :id)", + new MapSqlParameterSource("queue", queue.stream().map(Enum::name).toList()) + .addValue("queuedAt", Timestamp.from(waiting.queuedAt())).addValue("id", waiting.id()), + Integer.class); + } + + /** + * The oldest import of a queue. + */ + public Optional findNext(Collection queue) { + return jdbc.queryForList("select id from game_import where status in (:queue) order by queued_at, id limit 1", + new MapSqlParameterSource("queue", queue.stream().map(Enum::name).toList()), Long.class) + .stream().findFirst(); + } + + /** + * Imports a worker runs whose last progress is before {@code before}, with their status. + */ + public Map findStalled(Instant before) { + Map stalled = new LinkedHashMap<>(); + jdbc.query("select id, status from game_import where status in ('MATCHING', 'SAVING') " + + "and progress_at < :before order by id", + new MapSqlParameterSource("before", Timestamp.from(before)), + rs -> { stalled.put(rs.getLong("id"), GameImport.Status.valueOf(rs.getString("status"))); }); + return stalled; + } + + public long findUserId(long id) { + return jdbc.queryForObject("select user_id from game_import where id = :id", new MapSqlParameterSource("id", id), + Long.class); + } + + /** + * Marks the import as run by a worker, from now on watched by the watchdog. + */ + public void start(long id, GameImport.Status status) { + jdbc.update("update game_import set status = :status, progress_at = now() where id = :id", + new MapSqlParameterSource("id", id).addValue("status", status.name())); + } + + public void setStep(long id, GameImport.Step step) { + jdbc.update("update game_import set step = :step, progress_at = now() where id = :id", + new MapSqlParameterSource("id", id).addValue("step", step.name())); + } + + public void markReady(long id) { + jdbc.update("update game_import set status = 'READY', progress_at = null where id = :id", + new MapSqlParameterSource("id", id)); + } + + /** + * After the games are added: a status change after saved_at is the user's. + */ + public void markSaved(long id) { + jdbc.update("update game_import set status = 'SAVED', saved_at = now(), step = null, progress_at = null " + + "where id = :id", new MapSqlParameterSource("id", id)); + } + + /** + * Counts a stopped try: back to the end of the queue of its phase, or FAILED after {@code maxAttempts} tries. + */ + public void retryOrFail(long id, int maxAttempts) { + jdbc.update(""" + update game_import set attempts = attempts + 1, queued_at = now(), progress_at = null, step = null, + status = case when attempts + 1 >= :max then 'FAILED' + when status in ('QUEUED', 'MATCHING') then 'QUEUED' else 'SAVING' end + where id = :id and status in ('QUEUED', 'MATCHING', 'SAVING') + """, new MapSqlParameterSource("id", id).addValue("max", maxAttempts)); + } + + /** + * A FAILED import goes back to the queue of the phase it stopped in, with new tries. In the matching queue the + * V22 index refuses a second import per user with a DuplicateKeyException. + * + * @return false when the import did not fail + */ + public boolean retry(long userId, long id) { + return jdbc.update(""" + update game_import i set attempts = 0, queued_at = now(), progress_at = null, + status = case when exists (select 1 from game_import_row r + where r.import_id = i.id and r.chosen_igdb_id is not null) + then 'SAVING' else 'QUEUED' end + where id = :id and user_id = :user and status = 'FAILED' + """, new MapSqlParameterSource("id", id).addValue("user", userId)) == 1; + } + + /** + * @return false when the import was in none of {@code from} (a second save, or an undo of an import not saved) + */ + public boolean changeStatus(long userId, long id, Collection from, GameImport.Status to) { + return jdbc.update("update game_import set status = :to, queued_at = now() " + + "where id = :id and user_id = :user and status in (:from)", + new MapSqlParameterSource("id", id).addValue("user", userId) + .addValue("from", from.stream().map(Enum::name).toList()).addValue("to", to.name())) == 1; + } + + public List findUnmatchedLines(long importId) { + return jdbc.query("select * from game_import_row where import_id = :id and matched_at is null order by line_no", + new MapSqlParameterSource("id", importId), (rs, row) -> line(rs)); + } + + public List findRows(long importId) { + return jdbc.query("select * from game_import_row where import_id = :id order by line_no", + new MapSqlParameterSource("id", importId), (rs, row) -> new GameImport.Row(line(rs), + rs.getBoolean("search_failed"), rs.getBoolean("exact"), + jsonMapper.readValue(rs.getString("candidates"), + jsonMapper.getTypeFactory().constructCollectionType(List.class, IgdbApiService.Game.class)))); + } + + public void saveMatch(long importId, int lineNo, boolean searchFailed, boolean exact, + List candidates) { + jdbc.update("update game_import_row set matched_at = now(), search_failed = :failed, exact = :exact, " + + "candidates = cast(:candidates as jsonb) where import_id = :id and line_no = :line", + new MapSqlParameterSource("id", importId).addValue("line", lineNo).addValue("failed", searchFailed) + .addValue("exact", exact).addValue("candidates", jsonMapper.writeValueAsString(candidates))); + touch(importId); + } + + /** + * Stores the review; a line number the import does not have is left out. + */ + public void saveChoices(long importId, List choices) { + jdbc.batchUpdate("update game_import_row set chosen_igdb_id = :igdb, chosen_platform = :platform, " + + "chosen_status = :status, chosen_minutes = :minutes where import_id = :id and line_no = :line", + choices.stream().map(choice -> new MapSqlParameterSource("id", importId) + .addValue("line", choice.lineNo()).addValue("igdb", choice.igdbId()) + .addValue("platform", choice.platform()).addValue("status", choice.status().name()) + .addValue("minutes", choice.playtimeMinutes())).toArray(MapSqlParameterSource[]::new)); + } + + /** + * The chosen lines the save has not done yet, in line order. + */ + public List findUnsavedChoices(long importId) { + return jdbc.query("select * from game_import_row where import_id = :id and chosen_igdb_id is not null " + + "and save_result is null order by line_no", + new MapSqlParameterSource("id", importId), (rs, row) -> new GameImport.Choice(rs.getInt("line_no"), + rs.getLong("chosen_igdb_id"), rs.getString("chosen_platform"), + GameStatus.valueOf(rs.getString("chosen_status")), rs.getInt("chosen_minutes"))); + } + + public void saveResult(long importId, int lineNo, GameImport.SaveResult result) { + jdbc.update("update game_import_row set save_result = :result where import_id = :id and line_no = :line", + new MapSqlParameterSource("id", importId).addValue("line", lineNo).addValue("result", result.name())); + touch(importId); + } + + /** + * The import's games still added by hand; a game the Steam sync took over stays in the library. + */ + public int countGames(long userId, long importId) { + return jdbc.queryForObject("select count(*) from user_game where user_id = :user and import_id = :import " + + "and source = 'MANUAL'", new MapSqlParameterSource("user", userId).addValue("import", importId), + Integer.class); + } + + public List findAddedAppids(long importId) { + return jdbc.queryForList("select appid from user_game where import_id = :import", + new MapSqlParameterSource("import", importId), Long.class); + } + + /** + * Of {@link #countGames}: those with a rating, notes, priority or a status change after the save. + */ + public int countChangedGames(long userId, long importId) { + return jdbc.queryForObject(""" + select count(*) from user_game g join game_import i on i.id = g.import_id + where g.user_id = :user and g.import_id = :import and g.source = 'MANUAL' + and (g.rating is not null or g.notes is not null or g.priority is not null + or exists (select 1 from user_game_status_change c where c.user_id = g.user_id + and c.appid = g.appid and c.changed_at > i.saved_at)) + """, new MapSqlParameterSource("user", userId).addValue("import", importId), Integer.class); + } + + /** + * Deletes {@link #countGames} the way the manual remove does: with status, rating, notes and history. + */ + public int deleteGames(long userId, long importId) { + return jdbc.update("delete from user_game where user_id = :user and import_id = :import and source = 'MANUAL'", + new MapSqlParameterSource("user", userId).addValue("import", importId)); + } + + // The watchdog's sign of life. + private void touch(long importId) { + jdbc.update("update game_import set progress_at = now() where id = :id", new MapSqlParameterSource("id", importId)); + } + + private static GameImport.Line line(ResultSet rs) throws SQLException { + String status = rs.getString("status"); + return new GameImport.Line(rs.getInt("line_no"), rs.getString("title"), rs.getString("platform"), + rs.getObject("playtime_minutes", Integer.class), status != null ? GameStatus.valueOf(status) : null, + rs.getObject("igdb_id", Long.class)); + } + + private static Instant instant(Timestamp timestamp) { + return timestamp != null ? timestamp.toInstant() : null; + } +} diff --git a/src/main/java/core_service/modules/library/service/GameImportService.java b/src/main/java/core_service/modules/library/service/GameImportService.java new file mode 100644 index 0000000..81d1001 --- /dev/null +++ b/src/main/java/core_service/modules/library/service/GameImportService.java @@ -0,0 +1,173 @@ +package core_service.modules.library.service; + +import core_service.modules.igdb_adapter.service.IgdbApiService; +import core_service.modules.library.api.dto.GameImportDto; +import core_service.modules.library.api.dto.ImportRequest; +import core_service.modules.library.api.dto.ImportSaveRequest; +import core_service.modules.library.api.dto.UndoCheckDto; +import core_service.modules.library.domain.GameImport; +import core_service.modules.library.repo.GameImportRepository; +import core_service.shared.security.RequiresRole; +import core_service.shared.security.Role; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.IntStream; + +/** + * Imports of many games at once, see {@link GameImport}. The matching and the save run in {@link GameImportWorker}; + * the save adds each chosen line the way {@link ManualGameService#add} does. + */ +@Service +@RequiresRole(Role.USER) +public class GameImportService { + + public sealed interface CreateResult { + record Created(long id) implements CreateResult {} + + record AlreadyMatching() implements CreateResult {} + + record SearchUnavailable() implements CreateResult {} + } + + public enum ChangeResult { DONE, NOT_FOUND, WRONG_STATUS, ALREADY_MATCHING } + + private static final int HISTORY_SIZE = 20; + + private final GameImportRepository importRepository; + private final GameImportWorker worker; + private final ManualGameService manualGameService; + private final IgdbApiService igdbApiService; + private final TransactionTemplate transaction; + + public GameImportService(GameImportRepository importRepository, GameImportWorker worker, + ManualGameService manualGameService, IgdbApiService igdbApiService, + PlatformTransactionManager transactionManager) { + this.importRepository = importRepository; + this.worker = worker; + this.manualGameService = manualGameService; + this.igdbApiService = igdbApiService; + this.transaction = new TransactionTemplate(transactionManager); + } + + public CreateResult create(Long userId, GameImport.Source source, List rows) { + if (!igdbApiService.isEnabled()) { + return new CreateResult.SearchUnavailable(); + } + List lines = IntStream.range(0, rows.size()).mapToObj(index -> { + ImportRequest.Row row = rows.get(index); + return new GameImport.Line(index + 1, strippedOrNull(row.title()), strippedOrNull(row.platform()), + row.playtimeMinutes(), row.status(), row.igdbId()); + }).toList(); + long id; + try { + id = transaction.execute(tx -> importRepository.create(userId, source, lines)); + } catch (DuplicateKeyException e) { + return new CreateResult.AlreadyMatching(); + } + worker.wakeMatching(); + return new CreateResult.Created(id); + } + + public List list(Long userId) { + return importRepository.findByUser(userId, HISTORY_SIZE).stream().map(found -> dto(found, null)).toList(); + } + + public Optional get(Long userId, long id) { + return importRepository.find(userId, id).map(found -> dto(found, + found.status() == GameImport.Status.READY ? rows(userId, importRepository.findRows(id)) : null)); + } + + /** + * Once per import: stores the review and puts the import in the saving queue. + */ + public ChangeResult save(Long userId, long id, List chosen) { + if (importRepository.find(userId, id).isEmpty()) { + return ChangeResult.NOT_FOUND; + } + boolean queued = transaction.execute(tx -> { + if (!importRepository.changeStatus(userId, id, Set.of(GameImport.Status.READY), GameImport.Status.SAVING)) { + return false; + } + importRepository.saveChoices(id, chosen.stream().map(row -> new GameImport.Choice(row.lineNo(), + row.igdbId(), row.platform(), row.status(), + row.playtimeMinutes() != null ? row.playtimeMinutes() : 0)).toList()); + return true; + }); + if (!queued) { + return ChangeResult.WRONG_STATUS; + } + worker.wakeSaving(); + return ChangeResult.DONE; + } + + public ChangeResult retry(Long userId, long id) { + if (importRepository.find(userId, id).isEmpty()) { + return ChangeResult.NOT_FOUND; + } + try { + if (!importRepository.retry(userId, id)) { + return ChangeResult.WRONG_STATUS; + } + } catch (DuplicateKeyException e) { + return ChangeResult.ALREADY_MATCHING; + } + worker.wakeMatching(); + worker.wakeSaving(); + return ChangeResult.DONE; + } + + public Optional undoCheck(Long userId, long id) { + return importRepository.find(userId, id).map(found -> new UndoCheckDto( + importRepository.countGames(userId, id), importRepository.countChangedGames(userId, id))); + } + + /** + * Removes the import's games that are still added by hand, like {@link ManualGameService#remove}. Also for an + * import that FAILED while saving. + */ + public ChangeResult undo(Long userId, long id) { + if (importRepository.find(userId, id).isEmpty()) { + return ChangeResult.NOT_FOUND; + } + return transaction.execute(tx -> { + if (!importRepository.changeStatus(userId, id, Set.of(GameImport.Status.SAVED, GameImport.Status.FAILED), + GameImport.Status.UNDONE)) { + return ChangeResult.WRONG_STATUS; + } + importRepository.deleteGames(userId, id); + return ChangeResult.DONE; + }); + } + + private List rows(Long userId, List rows) { + Set owned = manualGameService.ownedIgdbIds(userId, + rows.stream().flatMap(row -> row.candidates().stream()).toList()); + return rows.stream().map(row -> new GameImportDto.Row(row.line().lineNo(), row.line().title(), + row.line().platform(), row.line().playtimeMinutes(), row.line().status(), row.searchFailed(), + row.exact(), row.candidates().stream().map(game -> new GameImportDto.Candidate(game.igdbId(), + game.name(), game.year(), game.imageUrl(), ManualGameService.fittingPlatforms(game), + owned.contains(game.igdbId()))).toList())).toList(); + } + + private GameImportDto dto(GameImport gameImport, List rows) { + Integer ahead = switch (gameImport.status()) { + case QUEUED -> importRepository.countAhead(gameImport, GameImportWorker.MATCHING_QUEUE); + case SAVING -> gameImport.running() ? null + : importRepository.countAhead(gameImport, GameImportWorker.SAVING_QUEUE); + default -> null; + }; + return new GameImportDto(gameImport.id(), gameImport.source(), gameImport.status(), gameImport.total(), + gameImport.matched(), gameImport.chosen(), gameImport.saved(), gameImport.added(), ahead, + gameImport.step(), gameImport.attempts(), gameImport.createdAt(), gameImport.savedAt(), rows); + } + + private static String strippedOrNull(String text) { + return text == null || text.isBlank() ? null : text.strip(); + } +} diff --git a/src/main/java/core_service/modules/library/service/GameImportWorker.java b/src/main/java/core_service/modules/library/service/GameImportWorker.java new file mode 100644 index 0000000..31a84eb --- /dev/null +++ b/src/main/java/core_service/modules/library/service/GameImportWorker.java @@ -0,0 +1,267 @@ +package core_service.modules.library.service; + +import core_service.modules.igdb_adapter.service.IgdbApiService; +import core_service.modules.library.domain.GameImport; +import core_service.modules.library.repo.GameImportRepository; +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.LongConsumer; +import java.util.stream.Collectors; + +/** + * Runs imports in two queues, each on one thread for all users, with game_import as the queue: + *
    + *
  • matching: IGDB searches, so an import of 1000 lines never takes the whole IGDB budget from the interactive + * search;
    • + *
  • saving: adds the chosen games, then loads their time to beat and moods. A save never waits behind another + * user's big matching.
    • + *
+ * Each line is stored when done, so a stopped try goes on where it stopped. A watchdog stops a try without progress + * for {@code import-stall-minutes}; a restart counts as a stopped try. After {@link #MAX_ATTEMPTS} tries the import + * is FAILED. + */ +@Service +@Slf4j +public class GameImportWorker { + + static final int MAX_CANDIDATES = 5; + static final int MAX_ATTEMPTS = 3; + private static final int RESTART_WAIT_SECONDS = 30; + static final Set MATCHING_QUEUE = Set.of(GameImport.Status.QUEUED, GameImport.Status.MATCHING); + static final Set SAVING_QUEUE = Set.of(GameImport.Status.SAVING); + + private final GameImportRepository importRepository; + private final ManualGameService manualGameService; + private final IgdbApiService igdbApiService; + private final TimeToBeatService timeToBeatService; + private final MoodService moodService; + private final Duration stallTime; + // Only a try that started before this app is one a restart stopped. + private final Instant startedAt = Instant.now(); + private final Lane matching; + private final Lane saving; + + public GameImportWorker(GameImportRepository importRepository, ManualGameService manualGameService, + IgdbApiService igdbApiService, TimeToBeatService timeToBeatService, MoodService moodService, + @Value("${app.library.import-stall-minutes:5}") long stallMinutes) { + this.importRepository = importRepository; + this.manualGameService = manualGameService; + this.igdbApiService = igdbApiService; + this.timeToBeatService = timeToBeatService; + this.moodService = moodService; + this.stallTime = Duration.ofMinutes(stallMinutes); + this.matching = new Lane("game-import-match", MATCHING_QUEUE, this::match); + this.saving = new Lane("game-import-save", SAVING_QUEUE, this::save); + } + + public void wakeMatching() { + matching.wake(); + } + + public void wakeSaving() { + saving.wake(); + } + + /** + * Imports a worker ran when the app stopped count as a stopped try; then both queues start. + */ + @EventListener(ApplicationReadyEvent.class) + public void resume() { + importRepository.findStalled(startedAt).keySet().forEach(id -> importRepository.retryOrFail(id, MAX_ATTEMPTS)); + matching.wake(); + saving.wake(); + } + + /** + * Stops a try without progress: the import goes back to the end of its queue (or FAILED), and its queue gets a + * new thread. The old thread is interrupted and writes nothing more once it sees that. Also wakes both queues: + * a run that died on a database error left its queue idle. + */ + @Scheduled(fixedDelayString = "${app.library.import-watchdog-ms:60000}") + public void watchdog() { + Map stalled = importRepository.findStalled(Instant.now().minus(stallTime)); + if (!stalled.isEmpty()) { + log.warn("Imports {} made no progress for {}, trying again", stalled, stallTime); + stalled.keySet().forEach(id -> importRepository.retryOrFail(id, MAX_ATTEMPTS)); + } + if (stalled.containsValue(GameImport.Status.MATCHING)) { + matching.restart(); + } else { + matching.wake(); + } + if (stalled.containsValue(GameImport.Status.SAVING)) { + saving.restart(); + } else { + saving.wake(); + } + } + + @PreDestroy + void stop() { + matching.stop(); + saving.stop(); + } + + private void match(long importId) { + importRepository.start(importId, GameImport.Status.MATCHING); + for (GameImport.Line line : importRepository.findUnmatchedLines(importId)) { + Match found = match(line); + // An IGDB call cut by the watchdog or the shutdown is no failed search. + if (Thread.currentThread().isInterrupted()) { + return; + } + importRepository.saveMatch(importId, line.lineNo(), found.searchFailed(), found.exact(), found.candidates()); + } + importRepository.markReady(importId); + } + + private void save(long importId) { + importRepository.start(importId, GameImport.Status.SAVING); + List choices = importRepository.findUnsavedChoices(importId); + if (!choices.isEmpty()) { + long userId = importRepository.findUserId(importId); + Map rows = importRepository.findRows(importId).stream() + .collect(Collectors.toMap(row -> row.line().lineNo(), row -> row)); + for (GameImport.Choice choice : choices) { + IgdbApiService.Game game = rows.get(choice.lineNo()).candidates().stream() + .filter(candidate -> candidate.igdbId() == choice.igdbId()).findFirst() + // A game from a new search in the review. IGDB knew it minutes ago, so no answer is an IGDB + // failure: the try stops and the line waits for the next one. + .or(() -> igdbApiService.getGame(choice.igdbId())) + .orElseThrow(() -> new IllegalStateException("IGDB did not answer for game " + choice.igdbId())); + if (Thread.currentThread().isInterrupted()) { + return; + } + boolean added = manualGameService.addGame(userId, game, choice.platform(), choice.status(), + choice.playtimeMinutes(), importId) instanceof ManualGameService.AddResult.Added; + importRepository.saveResult(importId, choice.lineNo(), + added ? GameImport.SaveResult.ADDED : GameImport.SaveResult.SKIPPED); + } + } + importRepository.setStep(importId, GameImport.Step.DETAILS); + List appids = importRepository.findAddedAppids(importId); + try { + timeToBeatService.refresh(appids); + moodService.refresh(appids); + } catch (RuntimeException e) { + // The next sync or page view asks again. + log.warn("Time to beat or moods of import {} not loaded: {}", importId, e.getMessage()); + } + if (Thread.currentThread().isInterrupted()) { + return; + } + importRepository.markSaved(importId); + } + + private record Match(boolean searchFailed, boolean exact, List candidates) {} + + private Match match(GameImport.Line line) { + if (line.igdbId() != null) { + Optional game = igdbApiService.getGame(line.igdbId()); + return new Match(game.isEmpty(), game.isPresent(), game.stream().toList()); + } + return manualGameService.findGames(line.title()) + .map(found -> rank(line, found)) + .map(candidates -> new Match(false, + !candidates.isEmpty() && sameName(candidates.getFirst().name(), line.title()), candidates)) + .orElseGet(() -> new Match(true, false, List.of())); + } + + // Same name first, among them one on the line's platform; else IGDB's order. + static List rank(GameImport.Line line, List found) { + Comparator order = Comparator + .comparing((IgdbApiService.Game game) -> !sameName(game.name(), line.title())) + .thenComparing(game -> line.platform() == null + || game.platforms().stream().noneMatch(line.platform()::equalsIgnoreCase)); + return found.stream().sorted(order).limit(MAX_CANDIDATES).toList(); + } + + /** + * Case, accents, punctuation and marks like ™ ignored, as in the speedrun.com match. + */ + static boolean sameName(String a, String b) { + return SpeedrunMatch.normalize(a).equals(SpeedrunMatch.normalize(b)); + } + + /** + * One thread that runs the imports of a queue, oldest first, until the queue is empty. A wake while it runs + * leaves one more run waiting, so an import queued meanwhile is not missed. + */ + private final class Lane { + + private final String name; + private final Set queue; + private final LongConsumer work; + private volatile ExecutorService executor; + + Lane(String name, Set queue, LongConsumer work) { + this.name = name; + this.queue = queue; + this.work = work; + this.executor = newExecutor(); + } + + void wake() { + executor.execute(this::drain); + } + + // Waits a while for the old thread, which stops at its next check, so two threads do not run one import. + synchronized void restart() { + ExecutorService old = executor; + old.shutdownNow(); + try { + if (!old.awaitTermination(RESTART_WAIT_SECONDS, TimeUnit.SECONDS)) { + log.warn("Thread {} did not stop; a new one starts anyway", name); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + executor = newExecutor(); + wake(); + } + + void stop() { + executor.shutdownNow(); + } + + private void drain() { + Optional next; + while (!Thread.currentThread().isInterrupted() && (next = importRepository.findNext(queue)).isPresent()) { + long id = next.get(); + try { + work.accept(id); + } catch (RuntimeException e) { + if (Thread.currentThread().isInterrupted()) { + return; + } + // To the end of the queue, so the next import is not blocked by this one. + log.error("Import {} stopped", id, e); + importRepository.retryOrFail(id, MAX_ATTEMPTS); + } + } + } + + private ExecutorService newExecutor() { + // One run and at most one waiting run; more wakes add nothing. + return new ThreadPoolExecutor(1, 1, 0, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(1), + Thread.ofPlatform().name(name).daemon().factory(), new ThreadPoolExecutor.DiscardPolicy()); + } + } +} diff --git a/src/main/java/core_service/modules/library/service/ManualGameService.java b/src/main/java/core_service/modules/library/service/ManualGameService.java index ce261dd..0fdd17a 100644 --- a/src/main/java/core_service/modules/library/service/ManualGameService.java +++ b/src/main/java/core_service/modules/library/service/ManualGameService.java @@ -23,6 +23,7 @@ import org.springframework.transaction.support.TransactionTemplate; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -86,8 +87,7 @@ public class ManualGameService { if (!igdbApiService.isEnabled()) { return Optional.of(new GameSearchDto(false, List.of())); } - return igdbApiService.searchGames(query).map(found -> withTypoHelp(query, found)) - .map(games -> new GameSearchDto(true, hits(userId, query, games))); + return findGames(query).map(games -> new GameSearchDto(true, hits(userId, query, games))); } // Games already in ToBePlayed first, with their catalog name and image (an auditor's correction included); then @@ -166,19 +166,7 @@ public class ManualGameService { if (found.isEmpty()) { return new AddResult.UnknownGame(); } - IgdbApiService.Game game = found.get(); - AddResult result; - try { - result = addInTransaction(userId, game, platform, status, playtimeMinutes); - } catch (DataIntegrityViolationException e) { - // A parallel add, sync or merge changed the rows after the check. The second try sees it. - log.info("Add of igdbId={} for user={} raced another write, trying again", igdbId, userId); - try { - result = addInTransaction(userId, game, platform, status, playtimeMinutes); - } catch (DataIntegrityViolationException again) { - return new AddResult.AlreadyInLibrary(); - } - } + AddResult result = addGame(userId, found.get(), platform, status, playtimeMinutes, null); if (result instanceof AddResult.Added added) { timeToBeatService.refreshInBackground(List.of(added.appid())); moodService.refreshInBackground(List.of(added.appid())); @@ -186,8 +174,56 @@ public class ManualGameService { return result; } + /** + * {@link #add} of a game already loaded from IGDB, without the background refresh; an import adds many games + * and loads their time to beat and moods once. + * + * @param importId the import that adds the game, so its undo can remove it; null for a single add. A removed + * row that comes back keeps no import id. + */ + public AddResult addGame(Long userId, IgdbApiService.Game game, String platform, GameStatus status, + int playtimeMinutes, Long importId) { + AddResult result; + try { + result = addInTransaction(userId, game, platform.strip(), status, playtimeMinutes, importId); + } catch (DataIntegrityViolationException e) { + // A parallel add, sync or merge changed the rows after the check. The second try sees it. + log.info("Add of igdbId={} for user={} raced another write, trying again", game.igdbId(), userId); + try { + result = addInTransaction(userId, game, platform.strip(), status, playtimeMinutes, importId); + } catch (DataIntegrityViolationException again) { + return new AddResult.AlreadyInLibrary(); + } + } + return result; + } + + /** + * IGDB's hits for a title, with the same typo help as {@link #search}; empty when IGDB failed or is off. + */ + public Optional> findGames(String query) { + if (!igdbApiService.isEnabled()) { + return Optional.empty(); + } + return igdbApiService.searchGames(query).map(found -> withTypoHelp(query, found)); + } + + /** + * The IGDB ids of the games the user already has, under any of their keys (a merge included). + */ + public Set ownedIgdbIds(Long userId, Collection games) { + if (games.isEmpty()) { + return Set.of(); + } + Map targets = mergeRepository.findTargets(games.stream().flatMap(game -> ownKeys(game).stream()).toList()); + Set owned = new HashSet<>(userGameRepository.findActiveAppids(userId, + games.stream().flatMap(game -> possibleKeys(game, targets).stream()).distinct().toList())); + return games.stream().filter(game -> possibleKeys(game, targets).stream().anyMatch(owned::contains)) + .map(IgdbApiService.Game::igdbId).collect(Collectors.toSet()); + } + private AddResult addInTransaction(Long userId, IgdbApiService.Game game, String platform, GameStatus status, - int playtimeMinutes) { + int playtimeMinutes, Long importId) { return transaction.execute(tx -> { Instant now = Instant.now(); Map targets = mergeRepository.findTargets(ownKeys(game)); @@ -210,8 +246,12 @@ public class ManualGameService { revived.setRemovedAt(null); // Only a Steam game can be hidden. revived.setHiddenAt(null); + // The row holds rating, notes and history from before, so an undo of the import must not delete it. + revived.setImportId(null); } else { - userGameRepository.save(UserGame.manual(userId, appid, platform, status, playtimeMinutes, now)); + UserGame added = UserGame.manual(userId, appid, platform, status, playtimeMinutes, now); + added.setImportId(importId); + userGameRepository.save(added); } return new AddResult.Added(appid); }); @@ -263,7 +303,7 @@ public class ManualGameService { } // A longer name would not fit the column; with no platform left, the user types one. - private static List fittingPlatforms(IgdbApiService.Game game) { + static List fittingPlatforms(IgdbApiService.Game game) { return game.platforms().stream().filter(platform -> platform.length() <= ManualGameUpdate.MAX_PLATFORM_LENGTH).toList(); } diff --git a/src/main/resources/db/migration/V22__Imports.sql b/src/main/resources/db/migration/V22__Imports.sql new file mode 100644 index 0000000..7627949 --- /dev/null +++ b/src/main/resources/db/migration/V22__Imports.sql @@ -0,0 +1,51 @@ +-- Imports of many games at once (item 23): a list or a CSV, matched with IGDB in the background, then saved as +-- games added by hand. A game an import added keeps its import_id, so the import can be undone. +-- The table is also the queue of the two workers (GameImportWorker): +-- QUEUED -> MATCHING -> READY (the user reviews) -> SAVING -> SAVED (-> UNDONE), or FAILED after 3 tries. +-- queued_at orders each queue. progress_at is set while a worker runs the import and after every line; NULL while +-- it waits. attempts counts the tries a watchdog or a restart stopped. +CREATE TABLE game_import ( + id BIGSERIAL PRIMARY KEY, + user_id BIGINT NOT NULL, + source VARCHAR(16) NOT NULL, + status VARCHAR(16) NOT NULL, + total INT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + queued_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + progress_at TIMESTAMPTZ, + attempts INT NOT NULL DEFAULT 0, + -- While SAVING: NULL = adding the games, DETAILS = loading their time to beat and moods. + step VARCHAR(16), + saved_at TIMESTAMPTZ +); +CREATE INDEX game_import_user ON game_import (user_id); +CREATE INDEX game_import_queue ON game_import (status, queued_at); +-- One import in the matching queue per user; an import of 1000 lines takes minutes of the shared IGDB budget. +CREATE UNIQUE INDEX game_import_one_matching ON game_import (user_id) WHERE status IN ('QUEUED', 'MATCHING'); + +-- One row per line of the input. igdb_id: the game the user already picked in the search (source SEARCH). +-- matched_at NULL: not matched yet. The chosen_* columns hold the user's review, save_result what the save did. +CREATE TABLE game_import_row ( + import_id BIGINT NOT NULL REFERENCES game_import (id) ON DELETE CASCADE, + line_no INT NOT NULL, + title VARCHAR(200), + platform VARCHAR(32), + playtime_minutes INT, + status VARCHAR(16), + igdb_id BIGINT, + matched_at TIMESTAMPTZ, + search_failed BOOLEAN NOT NULL DEFAULT FALSE, + exact BOOLEAN NOT NULL DEFAULT FALSE, + -- The IGDB hits, best first, kept whole (IgdbApiService.Game), so the save needs no IGDB call. + candidates JSONB NOT NULL DEFAULT '[]', + chosen_igdb_id BIGINT, + chosen_platform VARCHAR(32), + chosen_status VARCHAR(16), + chosen_minutes INT, + -- ADDED or SKIPPED (in the library already, or IGDB does not know the game). + save_result VARCHAR(8), + PRIMARY KEY (import_id, line_no) +); + +ALTER TABLE user_game ADD COLUMN import_id BIGINT REFERENCES game_import (id) ON DELETE SET NULL; +CREATE INDEX user_game_import ON user_game (import_id) WHERE import_id IS NOT NULL; diff --git a/src/test/java/core_service/modules/account/repo/AccountCleanupRepositoryTest.java b/src/test/java/core_service/modules/account/repo/AccountCleanupRepositoryTest.java index 4bf788f..bbe80b2 100644 --- a/src/test/java/core_service/modules/account/repo/AccountCleanupRepositoryTest.java +++ b/src/test/java/core_service/modules/account/repo/AccountCleanupRepositoryTest.java @@ -17,7 +17,7 @@ class AccountCleanupRepositoryTest extends AbstractPersistenceTest { private static final List USER_TABLES = List.of("user_game_status_change", "user_game_achievement", "user_game_achievement_meta", "user_game_playtime_day", "wishlist_game", "backlog_snapshot", - "user_steam_sync", "user_game"); + "user_steam_sync", "user_game", "game_import"); @Autowired private AccountCleanupRepository repository; @@ -42,6 +42,7 @@ class AccountCleanupRepositoryTest extends AbstractPersistenceTest { jdbc.update(""" insert into user_steam_sync (user_id, steam_id, user_seen_at) values (?, '76561198000000001', now())""", userId); + jdbc.update("insert into game_import (user_id, source, status, total) values (?, 'TEXT', 'SAVED', 0)", userId); } } diff --git a/src/test/java/core_service/modules/library/api/GameImportControllerTest.java b/src/test/java/core_service/modules/library/api/GameImportControllerTest.java new file mode 100644 index 0000000..aab8a93 --- /dev/null +++ b/src/test/java/core_service/modules/library/api/GameImportControllerTest.java @@ -0,0 +1,410 @@ +package core_service.modules.library.api; + +import core_service.configuration.security.ClientHeaderFilter; +import core_service.modules.igdb_adapter.service.IgdbApiService; +import core_service.modules.library.domain.GameSource; +import core_service.modules.library.domain.GameStatus; +import core_service.modules.library.domain.UserGame; +import core_service.modules.library.domain.UserGameId; +import core_service.modules.library.repo.UserGameRepository; +import core_service.modules.library.service.GameImportWorker; +import core_service.shared.test.AbstractIntegrationTest; +import com.jayway.jsonpath.JsonPath; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc; +import org.springframework.http.MediaType; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.bean.override.mockito.MockitoBean; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.ResultActions; + +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * The dev profile authenticates every request as user 1. + */ +@AutoConfigureMockMvc +class GameImportControllerTest extends AbstractIntegrationTest { + + private static final long USER_ID = 1L; + private static final IgdbApiService.Game ZELDA = new IgdbApiService.Game(119388, "Zelda: Tears of the Kingdom", 2023, + null, List.of("Switch"), null); + private static final IgdbApiService.Game ELDEN_RING = new IgdbApiService.Game(119133, "Elden Ring", 2022, + null, List.of("PS5", "PC"), 1245620L); + private static final IgdbApiService.Game NIGHTREIGN = new IgdbApiService.Game(325591, "Elden Ring Nightreign", 2025, + null, List.of("PS5", "PC"), 2622380L); + + @Autowired + private MockMvc mockMvc; + + @Autowired + private UserGameRepository userGameRepository; + + @Autowired + private GameImportWorker worker; + + @Autowired + private JdbcTemplate jdbc; + + @MockitoBean + private IgdbApiService igdbApiService; + + @BeforeEach + void seed() { + jdbc.update("delete from game_alias"); + userGameRepository.deleteAll(); + jdbc.update("delete from game_import"); + when(igdbApiService.isEnabled()).thenReturn(true); + when(igdbApiService.searchGames("Elden Ring")).thenReturn(Optional.of(List.of(NIGHTREIGN, ELDEN_RING))); + when(igdbApiService.searchGames("Zelda Tears")).thenReturn(Optional.of(List.of(ZELDA))); + when(igdbApiService.getGame(119388)).thenReturn(Optional.of(ZELDA)); + when(igdbApiService.getTimeToBeat(any())).thenReturn(Optional.of(Map.of())); + when(igdbApiService.getTimeToBeatByGameIds(any())).thenReturn(Optional.of(Map.of())); + } + + private ResultActions create(String json) throws Exception { + return mockMvc.perform(post("/api/library/imports") + .header(ClientHeaderFilter.CLIENT_HEADER, "test") + .contentType(MediaType.APPLICATION_JSON) + .content(json)); + } + + private long createReady(String json) throws Exception { + long id = ((Number) JsonPath.read(create(json).andExpect(status().isAccepted()) + .andReturn().getResponse().getContentAsString(), "$.id")).longValue(); + awaitReady(id); + return id; + } + + private void awaitReady(long id) throws InterruptedException { + awaitStatus(id, "READY"); + } + + // The matching and the save run in the background. + private void awaitStatus(long id, String expected) throws InterruptedException { + for (int attempt = 0; attempt < 50; attempt++) { + if (expected.equals(jdbc.queryForObject("select status from game_import where id = ?", String.class, id))) { + return; + } + Thread.sleep(100); + } + throw new AssertionError("Import " + id + " not " + expected); + } + + private ResultActions save(long id, String rows) throws Exception { + return mockMvc.perform(post("/api/library/imports/" + id + "/save") + .header(ClientHeaderFilter.CLIENT_HEADER, "test") + .contentType(MediaType.APPLICATION_JSON) + .content("{\"rows\": [" + rows + "]}")); + } + + private void saveAndWait(long id, String rows) throws Exception { + save(id, rows).andExpect(status().isAccepted()); + awaitStatus(id, "SAVED"); + } + + // Not woken: no worker takes it unless the test wakes one. + private long insertImport(long userId, String status, String set) { + long id = jdbc.queryForObject("insert into game_import (user_id, source, status, total) " + + "values (?, 'TEXT', ?, 0) returning id", Long.class, userId, status); + if (!set.isEmpty()) { + jdbc.update("update game_import set " + set + " where id = ?", id); + } + return id; + } + + private UserGame stored(long appid) { + return userGameRepository.findById(new UserGameId(USER_ID, appid)).orElseThrow(); + } + + private Long importIdOf(long appid) { + return jdbc.queryForObject("select import_id from user_game where user_id = ? and appid = ?", Long.class, + USER_ID, appid); + } + + @Test + void create_MatchesInTheBackground_SameNameFirstAndMarksGamesInTheLibrary() throws Exception { + jdbc.update("insert into steam_game (appid, name) values (1245620, 'ELDEN RING') on conflict do nothing"); + userGameRepository.save(UserGame.manual(USER_ID, 1245620L, "PC", GameStatus.BACKLOG, 0, Instant.now())); + + long id = createReady("{\"source\": \"TEXT\", \"rows\": [{\"title\": \" Elden Ring \", \"platform\": \"PS5\"}, " + + "{\"title\": \"Unknown Game\"}]}"); + + mockMvc.perform(get("/api/library/imports/" + id)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.total").value(2)) + .andExpect(jsonPath("$.matched").value(2)) + .andExpect(jsonPath("$.rows[0].title").value("Elden Ring")) + .andExpect(jsonPath("$.rows[0].platform").value("PS5")) + .andExpect(jsonPath("$.rows[0].exact").value(true)) + .andExpect(jsonPath("$.rows[0].candidates[*].name").value(contains("Elden Ring", "Elden Ring Nightreign"))) + .andExpect(jsonPath("$.rows[0].candidates[*].inLibrary").value(contains(true, false))) + // Mockito answers every other search with empty: IGDB failed. + .andExpect(jsonPath("$.rows[1].searchFailed").value(true)) + .andExpect(jsonPath("$.rows[1].candidates").isEmpty()); + mockMvc.perform(get("/api/library/imports")) + .andExpect(jsonPath("$.imports[0].id").value(id)) + .andExpect(jsonPath("$.imports[0].status").value("READY")) + .andExpect(jsonPath("$.imports[0].rows").value(nullValue())); + } + + @Test + void create_SearchRow_TakesThePickedGameWithoutASearch() throws Exception { + long id = createReady("{\"source\": \"SEARCH\", \"rows\": [{\"igdbId\": 119388}]}"); + + mockMvc.perform(get("/api/library/imports/" + id)) + .andExpect(jsonPath("$.rows[0].exact").value(true)) + .andExpect(jsonPath("$.rows[0].candidates[0].igdbId").value(119388)); + verify(igdbApiService, never()).searchGames(any()); + } + + @Test + void create_TooManyRowsOrNoTitle_Is400() throws Exception { + String row = "{\"title\": \"Game\"}"; + create("{\"source\": \"TEXT\", \"rows\": [" + String.join(",", Collections.nCopies(1001, row)) + "]}") + .andExpect(status().isBadRequest()); + create("{\"source\": \"TEXT\", \"rows\": [{\"platform\": \"PC\"}]}").andExpect(status().isBadRequest()); + } + + @Test + void create_WhileAnotherImportIsMatching_Is409() throws Exception { + jdbc.update("insert into game_import (user_id, source, status, total) values (?, 'TEXT', 'MATCHING', 0)", USER_ID); + + create("{\"source\": \"TEXT\", \"rows\": [{\"title\": \"Elden Ring\"}]}") + .andExpect(status().isConflict()) + .andExpect(jsonPath("$.code").value("IMPORT_RUNNING")); + } + + @Test + void get_ImportOfAnotherUser_Is404() throws Exception { + Long id = jdbc.queryForObject("insert into game_import (user_id, source, status, total) " + + "values (2, 'TEXT', 'READY', 0) returning id", Long.class); + + mockMvc.perform(get("/api/library/imports/" + id)).andExpect(status().isNotFound()); + save(id, "").andExpect(status().isNotFound()); + mockMvc.perform(delete("/api/library/imports/" + id + "/games").header(ClientHeaderFilter.CLIENT_HEADER, "test")) + .andExpect(status().isNotFound()); + } + + @Test + void save_AddsTheCheckedLinesWithTheImportId_SkipsDuplicatesAndOnlyOnce() throws Exception { + long id = createReady("{\"source\": \"CSV\", \"rows\": [{\"title\": \"Elden Ring\"}, {\"title\": \"Elden Ring\"}]}"); + when(igdbApiService.getGame(anyLong())).thenReturn(Optional.empty()); + + saveAndWait(id, "{\"lineNo\": 1, \"igdbId\": 119133, \"platform\": \" PS5 \", \"status\": \"PLAYING\", \"playtimeMinutes\": 90}," + + "{\"lineNo\": 2, \"igdbId\": 119133, \"platform\": \"PC\", \"status\": \"BACKLOG\"}," + + "{\"lineNo\": 9, \"igdbId\": 119133, \"platform\": \"PC\", \"status\": \"BACKLOG\"}"); + + mockMvc.perform(get("/api/library/imports/" + id)) + .andExpect(jsonPath("$.chosen").value(2)) + .andExpect(jsonPath("$.saved").value(2)) + .andExpect(jsonPath("$.added").value(1)) + .andExpect(jsonPath("$.step").value(nullValue())) + .andExpect(jsonPath("$.savedAt").isNotEmpty()); + + UserGame game = stored(1245620L); + assertEquals(GameSource.MANUAL, game.getSource()); + assertEquals("PS5", game.getPlatform()); + assertEquals(GameStatus.PLAYING, game.getStatus()); + assertEquals(90, game.getPlaytimeMinutes()); + assertEquals(id, importIdOf(1245620L)); + // The candidate was stored whole: no IGDB call for the save. + verify(igdbApiService, never()).getGame(119133); + save(id, "").andExpect(status().isConflict()); + } + + @Test + void save_GameFromANewSearch_IsReadFromIgdb() throws Exception { + long id = createReady("{\"source\": \"TEXT\", \"rows\": [{\"title\": \"Elden Ring\"}]}"); + + saveAndWait(id, "{\"lineNo\": 1, \"igdbId\": 119388, \"platform\": \"Switch\", \"status\": \"BACKLOG\"}"); + + assertEquals(id, importIdOf(-119388L)); + } + + @Test + void undo_RemovesTheGamesStillAddedByHand_AndCountsTheChangedOnes() throws Exception { + long id = createReady("{\"source\": \"TEXT\", \"rows\": [{\"title\": \"Elden Ring\"}, {\"title\": \"Zelda Tears\"}]}"); + saveAndWait(id, "{\"lineNo\": 1, \"igdbId\": 119133, \"platform\": \"PC\", \"status\": \"BACKLOG\"}," + + "{\"lineNo\": 2, \"igdbId\": 119388, \"platform\": \"Switch\", \"status\": \"BACKLOG\"}"); + mockMvc.perform(get("/api/library/imports/" + id + "/undo-check")) + .andExpect(jsonPath("$.games").value(2)) + .andExpect(jsonPath("$.changed").value(0)); + + jdbc.update("update user_game set status = 'PLAYING', status_set_by_user = true where appid = -119388"); + mockMvc.perform(get("/api/library/imports/" + id + "/undo-check")) + .andExpect(jsonPath("$.changed").value(1)); + // The Steam sync took Elden Ring over: the user owns it on Steam now. + jdbc.update("update user_game set source = 'STEAM' where appid = 1245620"); + mockMvc.perform(get("/api/library/imports/" + id + "/undo-check")) + .andExpect(jsonPath("$.games").value(1)); + + mockMvc.perform(delete("/api/library/imports/" + id + "/games").header(ClientHeaderFilter.CLIENT_HEADER, "test")) + .andExpect(status().isNoContent()); + + assertTrue(userGameRepository.findById(new UserGameId(USER_ID, -119388L)).isEmpty()); + assertEquals(GameSource.STEAM, stored(1245620L).getSource()); + assertEquals("UNDONE", jdbc.queryForObject("select status from game_import where id = ?", String.class, id)); + mockMvc.perform(delete("/api/library/imports/" + id + "/games").header(ClientHeaderFilter.CLIENT_HEADER, "test")) + .andExpect(status().isConflict()); + } + + @Test + void resume_CountsTheStoppedTryAndGoesOnWhereItStopped() throws Exception { + // Before this app started: a try the restart stopped. + long id = insertImport(USER_ID, "MATCHING", "progress_at = now() - interval '1 day'"); + jdbc.update("insert into game_import_row (import_id, line_no, title, matched_at, exact) " + + "values (?, 1, 'Done before', now(), true)", id); + jdbc.update("insert into game_import_row (import_id, line_no, title) values (?, 2, 'Elden Ring')", id); + + worker.resume(); + awaitReady(id); + + verify(igdbApiService, never()).searchGames("Done before"); + assertEquals(2, jdbc.queryForObject("select jsonb_array_length(candidates) from game_import_row " + + "where import_id = ? and line_no = 2", Integer.class, id)); + assertEquals(1, jdbc.queryForObject("select attempts from game_import where id = ?", Integer.class, id)); + } + + @Test + void watchdog_StopsATryWithoutProgress_AndFailsAfterThreeTries() throws Exception { + long stalled = insertImport(USER_ID, "MATCHING", "progress_at = now() - interval '10 minutes'"); + long lastTry = insertImport(2L, "SAVING", "progress_at = now() - interval '10 minutes', attempts = 2"); + long fine = insertImport(3L, "MATCHING", "progress_at = now()"); + + worker.watchdog(); + + awaitReady(stalled); + assertEquals(1, jdbc.queryForObject("select attempts from game_import where id = ?", Integer.class, stalled)); + assertEquals("FAILED", jdbc.queryForObject("select status from game_import where id = ?", String.class, lastTry)); + assertEquals(0, jdbc.queryForObject("select attempts from game_import where id = ?", Integer.class, fine)); + } + + @Test + void retry_PutsAFailedImportBackInTheQueueOfItsPhase() throws Exception { + long id = insertImport(USER_ID, "FAILED", "attempts = 3"); + jdbc.update("insert into game_import_row (import_id, line_no, title) values (?, 1, 'Elden Ring')", id); + + mockMvc.perform(post("/api/library/imports/" + id + "/retry").header(ClientHeaderFilter.CLIENT_HEADER, "test")) + .andExpect(status().isAccepted()); + + awaitReady(id); + assertEquals(0, jdbc.queryForObject("select attempts from game_import where id = ?", Integer.class, id)); + mockMvc.perform(post("/api/library/imports/" + id + "/retry").header(ClientHeaderFilter.CLIENT_HEADER, "test")) + .andExpect(status().isConflict()) + .andExpect(jsonPath("$.code").value("IMPORT_NOT_FAILED")); + } + + @Test + void get_WaitingImport_ShowsHowManyImportsAreBeforeIt() throws Exception { + insertImport(2L, "QUEUED", "queued_at = now() - interval '2 minutes'"); + insertImport(3L, "MATCHING", "queued_at = now() - interval '1 minutes', progress_at = now()"); + long mine = insertImport(USER_ID, "QUEUED", ""); + long saving = insertImport(USER_ID, "SAVING", ""); + + mockMvc.perform(get("/api/library/imports/" + mine)) + .andExpect(jsonPath("$.status").value("QUEUED")) + .andExpect(jsonPath("$.ahead").value(2)); + mockMvc.perform(get("/api/library/imports/" + saving)).andExpect(jsonPath("$.ahead").value(0)); + jdbc.update("delete from game_import"); + } + + @Test + void list_AnswersAnObjectWithTheImports() throws Exception { + long id = insertImport(USER_ID, "READY", ""); + + mockMvc.perform(get("/api/library/imports")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.imports[0].id").value(id)); + } + + // A lane can start before ApplicationReadyEvent (an early create, the first watchdog run). + @Test + void resume_DoesNotCountAnImportALaneRunsNow() throws Exception { + CountDownLatch release = new CountDownLatch(1); + when(igdbApiService.searchGames("Slow Game")).thenAnswer(invocation -> { + release.await(5, TimeUnit.SECONDS); + return Optional.of(List.of()); + }); + long id = insertImport(USER_ID, "QUEUED", ""); + jdbc.update("insert into game_import_row (import_id, line_no, title) values (?, 1, 'Slow Game')", id); + worker.wakeMatching(); + awaitStatus(id, "MATCHING"); + + try { + worker.resume(); + } finally { + release.countDown(); + } + + awaitReady(id); + assertEquals(0, jdbc.queryForObject("select attempts from game_import where id = ?", Integer.class, id)); + } + + @Test + void save_RevivedRowOfTheLibrary_IsNotRemovedByTheUndo() throws Exception { + jdbc.update("insert into steam_game (appid, name) values (1245620, 'ELDEN RING') on conflict do nothing"); + jdbc.update("insert into user_game (user_id, appid, status, rating, notes, source, removed_at) " + + "values (?, 1245620, 'FINISHED', 5, 'Loved it', 'STEAM', now())", USER_ID); + long id = createReady("{\"source\": \"TEXT\", \"rows\": [{\"title\": \"Elden Ring\"}]}"); + + saveAndWait(id, "{\"lineNo\": 1, \"igdbId\": 119133, \"platform\": \"PS5\", \"status\": \"PLAYING\"}"); + mockMvc.perform(delete("/api/library/imports/" + id + "/games").header(ClientHeaderFilter.CLIENT_HEADER, "test")) + .andExpect(status().isNoContent()); + + UserGame kept = stored(1245620L); + assertEquals(5, kept.getRating()); + assertEquals("Loved it", kept.getNotes()); + assertNull(importIdOf(1245620L)); + } + + @Test + void save_IgdbFailsForAPickFromANewSearch_KeepsTheLineForTheNextTry() throws Exception { + long id = createReady("{\"source\": \"TEXT\", \"rows\": [{\"title\": \"Elden Ring\"}]}"); + when(igdbApiService.getGame(119388)).thenReturn(Optional.empty()); + + save(id, "{\"lineNo\": 1, \"igdbId\": 119388, \"platform\": \"Switch\", \"status\": \"BACKLOG\"}") + .andExpect(status().isAccepted()); + awaitStatus(id, "FAILED"); + + assertNull(jdbc.queryForObject("select save_result from game_import_row where import_id = ?", String.class, id)); + assertEquals(3, jdbc.queryForObject("select attempts from game_import where id = ?", Integer.class, id)); + } + + @Test + void undo_ImportThatFailedWhileSaving_RemovesTheGamesItAdded() throws Exception { + long id = insertImport(USER_ID, "FAILED", "attempts = 3"); + jdbc.update("insert into steam_game (appid, name) values (-119388, 'Zelda') on conflict do nothing"); + jdbc.update("insert into user_game (user_id, appid, status, source, platform, import_id) " + + "values (?, -119388, 'BACKLOG', 'MANUAL', 'Switch', ?)", USER_ID, id); + + mockMvc.perform(delete("/api/library/imports/" + id + "/games").header(ClientHeaderFilter.CLIENT_HEADER, "test")) + .andExpect(status().isNoContent()); + + assertTrue(userGameRepository.findById(new UserGameId(USER_ID, -119388L)).isEmpty()); + } +} diff --git a/src/test/java/core_service/modules/library/service/GameImportWorkerTest.java b/src/test/java/core_service/modules/library/service/GameImportWorkerTest.java new file mode 100644 index 0000000..a2961a0 --- /dev/null +++ b/src/test/java/core_service/modules/library/service/GameImportWorkerTest.java @@ -0,0 +1,72 @@ +package core_service.modules.library.service; + +import core_service.modules.igdb_adapter.service.IgdbApiService; +import core_service.modules.library.domain.GameImport; +import core_service.modules.library.repo.GameImportRepository; +import org.junit.jupiter.api.Test; +import org.springframework.dao.DataAccessResourceFailureException; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class GameImportWorkerTest { + + private static IgdbApiService.Game game(long igdbId, String name, String... platforms) { + return new IgdbApiService.Game(igdbId, name, null, null, List.of(platforms), null); + } + + private static GameImport.Line line(String title, String platform) { + return new GameImport.Line(1, title, platform, null, null, null); + } + + @Test + void sameName_IgnoresCaseAccentsPunctuationAndMarks() { + assertThat(GameImportWorker.sameName("The Witcher® 3: Wild Hunt", "the witcher 3 wild hunt")).isTrue(); + assertThat(GameImportWorker.sameName("Pokémon Legends: Arceus", "Pokemon Legends Arceus")).isTrue(); + assertThat(GameImportWorker.sameName("Doom", "Doom II")).isFalse(); + } + + @Test + void rank_SameNameFirst_ThenTheLinesPlatform_ThenIgdbsOrder_AtMostFive() { + List found = List.of(game(1, "Doom Eternal", "PC"), game(2, "DOOM", "PC"), + game(3, "Doom", "Switch"), game(4, "Doom 3"), game(5, "Doom 64"), game(6, "Doom II")); + + assertThat(GameImportWorker.rank(line("Doom", "switch"), found)).extracting(IgdbApiService.Game::igdbId) + .containsExactly(3L, 2L, 1L, 4L, 5L); + assertThat(GameImportWorker.rank(line("Doom", null), found)).extracting(IgdbApiService.Game::igdbId) + .startsWith(2L, 3L); + assertThat(GameImportWorker.rank(line("x", null), IntStream.range(0, 9) + .mapToObj(i -> game(i, "Game " + i)).toList())).hasSize(GameImportWorker.MAX_CANDIDATES); + } + + @Test + void watchdog_WakesALaneWhoseRunDiedOnADatabaseError() { + GameImportRepository repository = mock(GameImportRepository.class); + when(repository.findNext(GameImportWorker.MATCHING_QUEUE)) + .thenThrow(new DataAccessResourceFailureException("connection lost")) + .thenReturn(Optional.of(7L), Optional.empty()); + when(repository.findStalled(any())).thenReturn(Map.of()); + GameImportWorker worker = new GameImportWorker(repository, mock(ManualGameService.class), + mock(IgdbApiService.class), mock(TimeToBeatService.class), mock(MoodService.class), 5); + try { + worker.wakeMatching(); + verify(repository, timeout(2000)).findNext(GameImportWorker.MATCHING_QUEUE); + + worker.watchdog(); + + // Import 7 waits in QUEUED: no try of it is running, so no watchdog rule ever picks it up. + verify(repository, timeout(2000)).markReady(7L); + } finally { + worker.stop(); + } + } +} diff --git a/src/test/java/core_service/shared/test/AbstractIntegrationTest.java b/src/test/java/core_service/shared/test/AbstractIntegrationTest.java index 42cf2e9..eb9a7fd 100644 --- a/src/test/java/core_service/shared/test/AbstractIntegrationTest.java +++ b/src/test/java/core_service/shared/test/AbstractIntegrationTest.java @@ -18,10 +18,12 @@ import java.util.Base64; * Full Spring context with Testcontainers. */ @Tag("IntegrationTest") -// The nightly sync would call Steam if a test ran at 23:55; speedrun.com needs no key, so it is switched off. +// The nightly sync would call Steam if a test ran at 23:55; speedrun.com needs no key, so it is switched off. The +// import watchdog runs only when a test calls it. // Every cached context keeps its own pool: small pools keep them all below Postgres' 100 clients. @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = {"app.library.nightly-sync-cron=-", "app.library.image-repair-cron=-", "app.speedrun.enabled=false", + "app.library.import-watchdog-ms=86400000", "spring.datasource.hikari.maximum-pool-size=4"}) @ActiveProfiles("dev") public abstract class AbstractIntegrationTest {