Merge pull request #67 from skettiks/recommendation-system-2

feat: improve recommendation system

src/main/kotlin/com/project/movienight/application/services/RecommendationService.kt

@@ -281,23 +281,28 @@ class RecommendationService(
         libraryEntries: List<FilmLibraryEntry>,
         filmsById: Map<UUID, Film>,
         weights: UserRecommendationWeights,
-    ): SparseVector {
+    ): UserTasteProfile {
-        val profile = MutableSparseVector()
+        val preferenceProfile = MutableSparseVector()
+        val positiveChoiceProfile = MutableSparseVector()
+        val negativeChoiceProfile = MutableSparseVector()
+        val libraryProfile = MutableSparseVector()
 
         preferences?.weightedGenres.orEmpty().forEach { (genre, weight) ->
-            profile.add(feature("genre", genre), weight.coerceAtLeast(1).toDouble() / MAX_PREFERENCE_WEIGHT)
+            preferenceProfile.add(feature("genre", genre), weight.coerceAtLeast(1).toDouble() / MAX_PREFERENCE_WEIGHT)
         }
         preferences?.plotTypes.orEmpty().forEach { plotType ->
-            tokenize(plotType).forEach { profile.add(feature("plot", it), PREFERENCE_PLOT_WEIGHT) }
+            tokenize(plotType).forEach { preferenceProfile.add(feature("plot", it), PREFERENCE_PLOT_WEIGHT) }
         }
-        preferences?.eras.orEmpty().forEach { profile.add(feature("era", it), PREFERENCE_ERA_WEIGHT) }
+        preferences?.eras.orEmpty().forEach { preferenceProfile.add(feature("era", it), PREFERENCE_ERA_WEIGHT) }
-        preferences?.castAndDirectors.orEmpty().forEach { profile.add(feature("person", it), PREFERENCE_PERSON_WEIGHT) }
+        preferences?.castAndDirectors.orEmpty().forEach {
-        preferences?.moods.orEmpty().forEach { profile.add(feature("mood", it), PREFERENCE_MOOD_WEIGHT) }
+            preferenceProfile.add(feature("person", it), PREFERENCE_PERSON_WEIGHT)
+        }
+        preferences?.moods.orEmpty().forEach { preferenceProfile.add(feature("mood", it), PREFERENCE_MOOD_WEIGHT) }
         preferences
             ?.contentTypes
             .orEmpty()
             .forEach {
-                profile.add(
+                preferenceProfile.add(
                     feature("type", it.name),
                     PREFERENCE_CONTENT_TYPE_WEIGHT,
                 )
@@ -306,42 +311,65 @@ class RecommendationService(
         ratings.forEach { rating ->
             val film = filmsById[rating.filmId] ?: return@forEach
             val signal = ratingSignal(rating.score)
-            profile.add(buildFilmVector(film, weights).scale(signal))
+            val filmVector = buildFilmVector(film, weights)
+            when {
+                signal >= POSITIVE_CHOICE_SIGNAL_THRESHOLD -> positiveChoiceProfile.add(filmVector.scale(signal))
+                signal <= NEGATIVE_CHOICE_SIGNAL_THRESHOLD -> negativeChoiceProfile.add(filmVector.scale(-signal))
+            }
         }
 
         libraryEntries.filterNot { it.isViewed }.forEach { entry ->
             val film = filmsById[entry.filmId] ?: return@forEach
-            profile.add(buildFilmVector(film, weights).scale(LIBRARY_SIGNAL_WEIGHT))
+            libraryProfile.add(buildFilmVector(film, weights).scale(LIBRARY_SIGNAL_WEIGHT))
         }
 
-        return profile.toSparseVector()
+        val overallProfile = MutableSparseVector()
+        overallProfile.add(preferenceProfile.toSparseVector())
+        overallProfile.add(positiveChoiceProfile.toSparseVector().scale(EXPLICIT_CHOICE_PROFILE_WEIGHT))
+        overallProfile.add(negativeChoiceProfile.toSparseVector().scale(-EXPLICIT_CHOICE_PROFILE_WEIGHT))
+        overallProfile.add(libraryProfile.toSparseVector())
+
+        return UserTasteProfile(
+            overall = overallProfile.toSparseVector(),
+            preferences = preferenceProfile.toSparseVector(),
+            positiveChoices = positiveChoiceProfile.toSparseVector(),
+            negativeChoices = negativeChoiceProfile.toSparseVector(),
+            library = libraryProfile.toSparseVector(),
+        )
     }
 
     private fun scoreFilm(
         film: Film,
         query: RecommendationQuery,
         preferences: UserPreferences?,
-        userProfile: SparseVector,
+        userProfile: UserTasteProfile,
         inLibrary: Boolean,
         weights: UserRecommendationWeights,
     ): ScoredRecommendation {
         val reasons = mutableListOf<String>()
         val filmVector = buildFilmVector(film, weights)
-        val preferenceScore = cosineSimilarity(userProfile, filmVector)
+        val relevanceBreakdown = relevanceScore(userProfile, filmVector)
+        val preferenceScore = relevanceBreakdown.combined
         val qualityScore = qualityScore(film)
-        val contextScore = contextScore(film, query, preferences)
+        val contextScore = contextScore(film, query, preferences, userProfile, preferenceScore)
         val noveltyScore = if (inLibrary) LIBRARY_NOVELTY_SCORE else CATALOG_NOVELTY_SCORE
         val diversityScore = diversityScore(film, preferences)
-        val score =
+        val rawScore =
             weights.relevanceWeight * preferenceScore +
                 weights.qualityWeight * qualityScore +
                 weights.contextWeight * contextScore +
                 weights.noveltyWeight * noveltyScore +
                 weights.diversityWeight * diversityScore
+        val score = rawScore - explicitChoiceMisfitPenalty(userProfile, relevanceBreakdown)
 
-        if (preferenceScore > STRONG_REASON_THRESHOLD) {
+        if (relevanceBreakdown.positiveSimilarity > EXPLICIT_CHOICE_REASON_THRESHOLD) {
+            reasons += "Similar to films you rated highly"
+        } else if (preferenceScore > STRONG_REASON_THRESHOLD) {
             reasons += "Similar to user preferences and rating history"
         }
+        matchingPositiveTasteTags(film, userProfile).take(MAX_REASON_ITEMS).forEach { tag ->
+            reasons += "Shares taste signal: ${tag.toReasonLabel()}"
+        }
         matchingGenres(film, preferences).take(MAX_REASON_ITEMS).forEach { genre ->
             reasons += "Matches preferred genre: $genre"
         }
@@ -385,11 +413,13 @@ class RecommendationService(
         val normalizedGenres = film.genres.map(::normalize).filter { it.isNotBlank() }
         val plotTokens = tokenize("${film.title} ${film.description}")
         val moods = inferredMoods(film)
+        val semanticTags = semanticTags(film)
         val people = (film.directors + film.cast).map(::normalize).filter { it.isNotBlank() }
 
         vector.add(feature("type", film.contentType.name), weights.contentTypeVectorWeight)
         distribute(vector, "genre", normalizedGenres, weights.genreVectorWeight)
         distribute(vector, "plot", plotTokens, weights.plotVectorWeight)
+        distribute(vector, "tag", semanticTags, weights.plotVectorWeight * SEMANTIC_TAG_VECTOR_WEIGHT_MULTIPLIER)
         distribute(vector, "mood", moods, weights.moodVectorWeight)
         film.releaseYear?.let { vector.add(feature("era", decadeOf(it)), weights.eraVectorWeight) }
         distribute(vector, "person", people, weights.peopleVectorWeight)
@@ -397,10 +427,75 @@ class RecommendationService(
         return vector.toSparseVector()
     }
 
+    private fun relevanceScore(
+        userProfile: UserTasteProfile,
+        filmVector: SparseVector,
+    ): RelevanceBreakdown {
+        val overallSimilarity = cosineSimilarity(userProfile.overall, filmVector)
+        val preferenceSimilarity = cosineSimilarity(userProfile.preferences, filmVector)
+        val positiveSimilarity = cosineSimilarity(userProfile.positiveChoices, filmVector).coerceAtLeast(0.0)
+        val negativeSimilarity = cosineSimilarity(userProfile.negativeChoices, filmVector).coerceAtLeast(0.0)
+        val librarySimilarity = cosineSimilarity(userProfile.library, filmVector).coerceAtLeast(0.0)
+
+        if (!userProfile.hasExplicitChoices) {
+            return RelevanceBreakdown(
+                combined = overallSimilarity,
+                positiveSimilarity = positiveSimilarity,
+            )
+        }
+
+        val positiveComponent =
+            if (userProfile.hasPositiveChoices) {
+                positiveSimilarity * POSITIVE_CHOICE_RELEVANCE_WEIGHT
+            } else {
+                0.0
+            }
+        val preferenceComponent = preferenceSimilarity.coerceAtLeast(0.0) * BROAD_PREFERENCE_RELEVANCE_WEIGHT
+        val libraryComponent =
+            if (userProfile.hasLibraryChoices) {
+                librarySimilarity * LIBRARY_CHOICE_RELEVANCE_WEIGHT
+            } else {
+                0.0
+            }
+        val fallbackComponent = overallSimilarity.coerceAtLeast(0.0) * OVERALL_RELEVANCE_FALLBACK_WEIGHT
+        val negativePenalty =
+            if (userProfile.hasNegativeChoices) {
+                negativeSimilarity * NEGATIVE_CHOICE_RELEVANCE_PENALTY
+            } else {
+                0.0
+            }
+
+        return RelevanceBreakdown(
+            combined =
+                (positiveComponent + preferenceComponent + libraryComponent + fallbackComponent - negativePenalty)
+                    .coerceIn(MIN_RELEVANCE_SCORE, MAX_RELEVANCE_SCORE),
+            positiveSimilarity = positiveSimilarity,
+        )
+    }
+
+    private fun explicitChoiceMisfitPenalty(
+        userProfile: UserTasteProfile,
+        relevanceBreakdown: RelevanceBreakdown,
+    ): Double {
+        if (!userProfile.hasPositiveChoices) {
+            return 0.0
+        }
+        val fit = relevanceBreakdown.positiveSimilarity
+        if (fit >= POSITIVE_CHOICE_SOFT_FIT_THRESHOLD) {
+            return 0.0
+        }
+        val missingFitRatio =
+            ((POSITIVE_CHOICE_SOFT_FIT_THRESHOLD - fit) / POSITIVE_CHOICE_SOFT_FIT_THRESHOLD)
+                .coerceIn(0.0, 1.0)
+        return EXPLICIT_CHOICE_MISFIT_MAX_PENALTY * missingFitRatio
+    }
+
     private fun contextScore(
         film: Film,
         query: RecommendationQuery,
         preferences: UserPreferences?,
+        userProfile: UserTasteProfile,
+        relevanceScore: Double,
     ): Double {
         var score = 0.0
         var checks = 0
@@ -426,7 +521,16 @@ class RecommendationService(
             }
         }
 
-        return if (checks == 0) BASE_CONTEXT_SCORE else score / checks
+        val baseScore = if (checks == 0) BASE_CONTEXT_SCORE else score / checks
+        if (!userProfile.hasExplicitChoices) {
+            return baseScore
+        }
+
+        val relevanceGate =
+            MIN_CONTEXT_RELEVANCE_GATE +
+                (MAX_CONTEXT_RELEVANCE_GATE - MIN_CONTEXT_RELEVANCE_GATE) *
+                relevanceScore.coerceIn(0.0, 1.0)
+        return baseScore * relevanceGate
     }
 
     private fun qualityScore(film: Film): Double {
@@ -435,7 +539,7 @@ class RecommendationService(
                 film.imdbRating?.let { normalizeRating(it) },
                 film.platformRating?.let { normalizeRating(it) },
             )
-        return normalizedRatings.averageOrNull() ?: BASE_QUALITY_SCORE
+        return normalizedRatings.averageOrNull() ?: UNKNOWN_QUALITY_SCORE
     }
 
     private fun diversityScore(
@@ -452,9 +556,9 @@ class RecommendationService(
         val filmGenres = film.genres.map(::normalize).toSet()
         return when {
             preferredGenres.isEmpty() -> BASE_DIVERSITY_SCORE
-            filmGenres.none { it in preferredGenres } -> HIGH_DIVERSITY_SCORE
+            filmGenres.none { it in preferredGenres } -> LOW_DIVERSITY_SCORE
-            filmGenres.size > 1 -> MEDIUM_DIVERSITY_SCORE
+            filmGenres.size > 1 -> HIGH_DIVERSITY_SCORE
-            else -> LOW_DIVERSITY_SCORE
+            else -> MEDIUM_DIVERSITY_SCORE
         }
     }
 
@@ -465,6 +569,25 @@ class RecommendationService(
             .keys
     }
 
+    private fun semanticTags(film: Film): Set<String> {
+        val text = normalize("${film.title} ${film.description} ${film.genres.joinToString(" ")}")
+        return semanticTagLexicon
+            .filterValues { keywords -> keywords.any { keyword -> text.contains(keyword) } }
+            .keys
+    }
+
+    private fun matchingPositiveTasteTags(
+        film: Film,
+        userProfile: UserTasteProfile,
+    ): List<String> {
+        if (!userProfile.hasPositiveChoices) {
+            return emptyList()
+        }
+        return semanticTags(film)
+            .filter { tag -> userProfile.positiveChoices.values.containsKey(feature("tag", tag)) }
+            .sorted()
+    }
+
     private fun matchingGenres(
         film: Film,
         preferences: UserPreferences?,
@@ -541,6 +664,10 @@ class RecommendationService(
             .trim()
             .lowercase(Locale.getDefault())
 
+    private fun String.toReasonLabel(): String =
+        split("-")
+            .joinToString(" ") { token -> token.replaceFirstChar { char -> char.titlecase(Locale.getDefault()) } }
+
     private fun cosineSimilarity(
         left: SparseVector,
         right: SparseVector,
@@ -579,6 +706,24 @@ class RecommendationService(
         val diversityScore: Double,
     )
 
+    private data class RelevanceBreakdown(
+        val combined: Double,
+        val positiveSimilarity: Double,
+    )
+
+    private data class UserTasteProfile(
+        val overall: SparseVector,
+        val preferences: SparseVector,
+        val positiveChoices: SparseVector,
+        val negativeChoices: SparseVector,
+        val library: SparseVector,
+    ) {
+        val hasPositiveChoices: Boolean = positiveChoices.values.isNotEmpty()
+        val hasNegativeChoices: Boolean = negativeChoices.values.isNotEmpty()
+        val hasLibraryChoices: Boolean = library.values.isNotEmpty()
+        val hasExplicitChoices: Boolean = hasPositiveChoices || hasNegativeChoices || hasLibraryChoices
+    }
+
     private data class ScoreContributions(
         val relevance: Double,
         val quality: Double,
@@ -636,19 +781,35 @@ class RecommendationService(
         private const val PREFERENCE_MOOD_WEIGHT = 0.8
         private const val PREFERENCE_CONTENT_TYPE_WEIGHT = 0.5
         private const val LIBRARY_SIGNAL_WEIGHT = 0.25
+        private const val EXPLICIT_CHOICE_PROFILE_WEIGHT = 1.8
+        private const val SEMANTIC_TAG_VECTOR_WEIGHT_MULTIPLIER = 0.9
 
         private const val LEARNING_RATE = 0.03
 
         private const val LIBRARY_NOVELTY_SCORE = 0.85
         private const val CATALOG_NOVELTY_SCORE = 0.65
         private const val BASE_CONTEXT_SCORE = 0.5
-        private const val BASE_QUALITY_SCORE = 0.5
+        private const val UNKNOWN_QUALITY_SCORE = 0.42
         private const val BASE_DIVERSITY_SCORE = 0.5
-        private const val HIGH_DIVERSITY_SCORE = 1.0
+        private const val HIGH_DIVERSITY_SCORE = 0.75
-        private const val MEDIUM_DIVERSITY_SCORE = 0.6
+        private const val MEDIUM_DIVERSITY_SCORE = 0.45
-        private const val LOW_DIVERSITY_SCORE = 0.3
+        private const val LOW_DIVERSITY_SCORE = 0.15
         private const val STRONG_REASON_THRESHOLD = 0.15
+        private const val EXPLICIT_CHOICE_REASON_THRESHOLD = 0.12
         private const val QUALITY_REASON_THRESHOLD = 0.75
+        private const val POSITIVE_CHOICE_SIGNAL_THRESHOLD = 0.4
+        private const val NEGATIVE_CHOICE_SIGNAL_THRESHOLD = -0.3
+        private const val POSITIVE_CHOICE_RELEVANCE_WEIGHT = 0.78
+        private const val BROAD_PREFERENCE_RELEVANCE_WEIGHT = 0.12
+        private const val LIBRARY_CHOICE_RELEVANCE_WEIGHT = 0.08
+        private const val OVERALL_RELEVANCE_FALLBACK_WEIGHT = 0.08
+        private const val NEGATIVE_CHOICE_RELEVANCE_PENALTY = 0.65
+        private const val MIN_RELEVANCE_SCORE = -1.0
+        private const val MAX_RELEVANCE_SCORE = 1.0
+        private const val MIN_CONTEXT_RELEVANCE_GATE = 0.35
+        private const val MAX_CONTEXT_RELEVANCE_GATE = 1.0
+        private const val POSITIVE_CHOICE_SOFT_FIT_THRESHOLD = 0.10
+        private const val EXPLICIT_CHOICE_MISFIT_MAX_PENALTY = 0.12
 
         private val tokenSeparatorRegex = Regex("[^\\p{L}0-9]+")
         private val stopWords =
@@ -670,5 +831,32 @@ class RecommendationService(
                 "romantic" to listOf("romance", "love", "relationship"),
                 "focused" to listOf("science", "mission", "detective", "investigation", "sci-fi"),
             )
+        private val semanticTagLexicon =
+            mapOf(
+                "magic-fantasy" to
+                    listOf(
+                        "magic",
+                        "magical",
+                        "wizard",
+                        "witch",
+                        "spell",
+                        "sorcer",
+                        "fantasy",
+                        "enchanted",
+                        "dragon",
+                    ),
+                "wizard-school" to listOf("wizard school", "magic school", "academy", "school of magic"),
+                "young-adult" to listOf("young", "teen", "teenage", "teenager", "student", "coming of age"),
+                "family-adventure" to listOf("family", "friendship", "friends", "adventure", "quest"),
+                "quest-adventure" to listOf("quest", "journey", "treasure", "relic", "map", "kingdom"),
+                "heist-crime" to listOf("heist", "thief", "robbery", "criminal", "crime", "gang"),
+                "space-opera" to listOf("space", "spaceship", "galaxy", "planet", "alien", "starship"),
+                "superhero" to listOf("superhero", "hero", "masked", "powers", "mutant"),
+                "martial-arts" to listOf("martial", "kung fu", "samurai", "ninja", "warrior", "sword"),
+                "war-epic" to listOf("war", "battle", "army", "soldier", "general", "rebel"),
+                "mystery-investigation" to listOf("mystery", "detective", "investigation", "secret", "clue"),
+                "dark-fantasy" to listOf("dark force", "curse", "underworld", "demon", "monster"),
+                "animated-anime" to listOf("animation", "animated", "anime"),
+            )
     }
 }

src/test/kotlin/com/project/movienight/RecommendationSmokeTest.kt

@@ -335,6 +335,107 @@ class RecommendationSmokeTest {
             }
     }
 
+    @Test
+    fun `should rank films similar to highly rated choices above broad onboarding matches`() {
+        mockMvc
+            .post("/api/users") {
+                contentType = MediaType.APPLICATION_JSON
+                content =
+                    objectMapper.writeValueAsString(
+                        CreateUserRequest(
+                            name = "Harry",
+                            email = "harry@example.com",
+                        ),
+                    )
+            }.andExpect {
+                status { isCreated() }
+            }
+
+        val userId =
+            UUID.fromString(
+                jdbcTemplate.queryForObject(
+                    "SELECT id FROM users WHERE email = ?",
+                    String::class.java,
+                    "harry@example.com",
+                ),
+            )
+
+        val likedFirstFilmId =
+            createFilm(
+                title = "Wizard School Stone",
+                description = "A young wizard discovers a magic school, spells, friendship, and a hidden dark force.",
+                releaseYear = 2001,
+                genres = listOf("Fantasy", "Adventure", "Family"),
+                imdbRating = 8.0,
+            )
+        val likedSecondFilmId =
+            createFilm(
+                title = "Chamber of Magic",
+                description =
+                    "Young friends return to a wizard school and uncover a secret chamber full of magical danger.",
+                releaseYear = 2002,
+                genres = listOf("Fantasy", "Adventure", "Family"),
+                imdbRating = 8.1,
+            )
+        val magicCandidateId =
+            createFilm(
+                title = "Sorcerer Academy",
+                description = "A teenage student joins an academy with friends and faces an enchanted threat.",
+                releaseYear = 2005,
+                genres = listOf("Fantasy", "Adventure", "Family"),
+                imdbRating = 7.0,
+            )
+        createFilm(
+            title = "Highway Strike",
+            description = "An elite agent chases criminals through explosions, heists, and street fights.",
+            releaseYear = 2005,
+            genres = listOf("Action"),
+            imdbRating = 9.4,
+        )
+
+        mockMvc
+            .put("/api/users/$userId/preferences") {
+                contentType = MediaType.APPLICATION_JSON
+                content =
+                    objectMapper.writeValueAsString(
+                        UpsertUserPreferencesRequest(
+                            weightedGenres = mapOf("Action" to 5),
+                            eras = listOf("2000s"),
+                            contentTypes = listOf("FILM"),
+                        ),
+                    )
+            }.andExpect {
+                status { isOk() }
+            }
+
+        listOf(likedFirstFilmId, likedSecondFilmId).forEach { filmId ->
+            mockMvc
+                .post("/api/users/$userId/ratings/films/$filmId") {
+                    contentType = MediaType.APPLICATION_JSON
+                    content = objectMapper.writeValueAsString(RateFilmRequest(score = 10, note = "Favorite"))
+                }.andExpect {
+                    status { isCreated() }
+                }
+
+            mockMvc
+                .post("/api/users/$userId/library/films/$filmId/viewed")
+                .andExpect {
+                    status { isOk() }
+                }
+        }
+
+        mockMvc
+            .get("/api/users/$userId/recommendations") {
+                param("contentType", "FILM")
+                param("limit", "2")
+            }.andExpect {
+                status { isOk() }
+                jsonPath("$[0].filmId") { value(magicCandidateId.toString()) }
+                jsonPath("$[0].reasons[0]") { value("Similar to films you rated highly") }
+                jsonPath("$[0].reasons[1]") { value("Shares taste signal: Family Adventure") }
+            }
+    }
+
     private fun cleanDatabase() {
         jdbcTemplate.execute("DELETE FROM recommendation_events")
         jdbcTemplate.execute("DELETE FROM user_recommendation_weights")
@@ -370,6 +471,8 @@ class RecommendationSmokeTest {
 
     private fun createFilm(
         title: String,
+        description: String = "$title description",
+        releaseYear: Int? = null,
         genres: List<String>,
         imdbRating: Double,
     ): UUID {
@@ -380,8 +483,9 @@ class RecommendationSmokeTest {
                     objectMapper.writeValueAsString(
                         CreateFilmRequest(
                             title = title,
-                            description = "$title description",
+                            description = description,
                             contentType = "FILM",
+                            releaseYear = releaseYear,
                             genres = genres,
                             imdbRating = imdbRating,
                         ),