reprööt deduplication

src/stores/fetchers/timeline_fetcher.js

@@ -54,25 +54,36 @@ const timelineFetcher = (timeline, argument, credentials) => {
 
     if (older && bottomedOut.value) return
     return fetchTimeline(args)
-      .then(({ data: statuses, pagination, timestamp }) => {
+      .then(({ data, pagination, timestamp }) => {
         // No statuses for timeline, ever.
-        if (timeline.order.length === 0 && statuses.length === 0) {
+        if (timeline.order.length === 0 && data.length === 0) {
           bottomedOut.value = true
         }
-        if (!older && statuses.length >= 20 && numStatusesBeforeFetch > 0) {
+        if (!older && data.length >= 20 && numStatusesBeforeFetch > 0) {
           useTimelinesStore().requireReload(timeline.name)
         }
 
-        if (older && statuses.length === 0) {
+        if (older && data.length === 0) {
           bottomedOut.value = true
         }
 
-        const processed = useStatusesStore()
-          .addNewStatuses({ statuses, timestamp })
-          .map(({ id }) => id)
+        const processed = useStatusesStore().addNewStatuses({
+          statuses: data,
+          timestamp,
+        })
+
+        const statuses = processed.map(({ id }) => id)
+
+        const repeats = processed
+          .filter(({ retweeted_status }) => Boolean(retweeted_status))
+          .map(({ id, retweeted_status: { id: repeatedId } }) => [
+            id,
+            repeatedId,
+          ])
 
         useTimelinesStore().addStatusesToTimeline(timeline.name, argument, {
-          statuses: processed,
+          statuses,
+          repeats,
           showImmediately,
           older,
           pagination,

src/stores/timelines.js

@@ -21,6 +21,11 @@ const emptyTl = (name, argument = null) => {
     // Statuses shown to user
     visibleStatusIds: new Set(),
 
+    // Tracked reprööts, used to avoid displaying same reprööt more than once
+    repeatedToRepeat: new Map(),
+    repeatToRepeated: new Map(),
+    ignoredIds: new Set(),
+
     // Number of statuses not shown yet
     newStatusCount: 0,
 
@@ -188,6 +193,7 @@ export const useTimelinesStore = defineStore('timelines', {
       timeline.order = []
       timeline.statusIds = new Set()
       timeline.visibleStatusIds = new Set()
+      timeline.ignoredIds = new Set()
       timeline.newStatusCount = 0
       timeline.maxId = ''
       timeline.minId = ''
@@ -252,6 +258,7 @@ export const useTimelinesStore = defineStore('timelines', {
       argument,
       {
         statuses,
+        repeats,
         showImmediately = false,
         noIdUpdate = false,
         pagination = {},
@@ -261,6 +268,8 @@ export const useTimelinesStore = defineStore('timelines', {
       if (statuses.length === 0) return
       const timeline = this[timelineName]
 
+      this.populateRepeats(timeline, repeats)
+
       // This makes sure that user timeline won't get data meant for other
       // user. I.e. opening different user profiles makes request which could
       // return data late after user already viewing different user profile
@@ -287,13 +296,18 @@ export const useTimelinesStore = defineStore('timelines', {
         timeline.statusIds.add(statusId)
 
         if (isNew) {
-          if (showImmediately) {
-            // Add it directly to the visibleStatuses, don't change
-            // newStatusCount
-            timeline.visibleStatusIds.add(statusId)
+          const seenBefore = this.checkSeenBefore(timeline, statusId)
+          if (!seenBefore) {
+            if (showImmediately) {
+              // Add it directly to the visibleStatuses, don't change
+              // newStatusCount
+              timeline.visibleStatusIds.add(statusId)
+            } else {
+              // Just change newStatuscount
+              timeline.newStatusCount += 1
+            }
           } else {
-            // Just change newStatuscount
-            timeline.newStatusCount += 1
+            timeline.ignoredIds.add(statusId)
           }
         }
       })
@@ -301,9 +315,43 @@ export const useTimelinesStore = defineStore('timelines', {
     onStreamMessage(timeline, argument, event) {
       this.addStatusesToTimeline(timeline, argument, {
         statuses: event.data.map(({ id }) => id),
+        repeats: event.data
+          .filter(({ retweeted_status }) => Boolean(retweeted_status))
+          .map(({ id, retweeted_status: { id: repeatedId } }) => [
+            id,
+            repeatedId,
+          ]),
       })
     },
 
+    // Reprööt handling
+    populateRepeats(timeline, repeats) {
+      // Starting from oldest
+      ;[...repeats].reverse().forEach(([repeatId, repeatedId]) => {
+        timeline.repeatToRepeated.set(repeatId, repeatedId)
+        const knownRepeats =
+          timeline.repeatedToRepeat.get(repeatedId) ?? new Set()
+        knownRepeats.add(repeatId)
+        timeline.repeatedToRepeat.set(repeatedId, knownRepeats)
+      })
+    },
+    checkSeenBefore(timeline, statusId) {
+      // Check if this is a reprööt
+      const repeatedStatusId = timeline.repeatToRepeated.get(statusId)
+      // Non-reprööts are never seen before
+      if (!repeatedStatusId) return false
+      // We've seen this status already directly
+      if (timeline.statusIds.has(repeatedStatusId)) return true
+
+      // Check for reprööts
+      const knownRepeats = timeline.repeatedToRepeat.get(repeatedStatusId)
+
+      // If it's the only reprööt then we've never seen post before
+      if (knownRepeats.size === 1) return false
+      // If we're working on oldest known reprööt then we've never seen it before
+      return first(knownRepeats) !== statusId
+    },
+
     // Poll & Push
     onStreamConnect(timeline) {
       console.debug('[Timelines] Stream connected', timeline)
@@ -391,7 +439,11 @@ export const useTimelinesStore = defineStore('timelines', {
       timeline.newStatusCount = 0
       timeline.order = timeline.order.slice(0, 50)
       timeline.statusIds = new Set([...timeline.order])
-      timeline.visibleStatusIds = new Set([...timeline.order])
+
+      // Reprööts CAN be present in order but some must be ignored
+      timeline.visibleStatusIds = new Set([
+        ...timeline.order.filter((id) => !timeline.ignoredIds.has(id)),
+      ])
       this.updateTimelineExtremes(timeline)
     },
     syncOrder(timeline) {