diff --git a/src/stores/fetchers/timeline_fetcher.js b/src/stores/fetchers/timeline_fetcher.js index 90a5f845a..494377b94 100644 --- a/src/stores/fetchers/timeline_fetcher.js +++ b/src/stores/fetchers/timeline_fetcher.js @@ -54,25 +54,36 @@ const timelineFetcher = (timeline, argument, credentials) => { if (older && bottomedOut.value) return return fetchTimeline(args) - .then(({ data: statuses, pagination, timestamp }) => { + .then(({ data, pagination, timestamp }) => { // No statuses for timeline, ever. - if (timeline.order.length === 0 && statuses.length === 0) { + if (timeline.order.length === 0 && data.length === 0) { bottomedOut.value = true } - if (!older && statuses.length >= 20 && numStatusesBeforeFetch > 0) { + if (!older && data.length >= 20 && numStatusesBeforeFetch > 0) { useTimelinesStore().requireReload(timeline.name) } - if (older && statuses.length === 0) { + if (older && data.length === 0) { bottomedOut.value = true } - const processed = useStatusesStore() - .addNewStatuses({ statuses, timestamp }) - .map(({ id }) => id) + const processed = useStatusesStore().addNewStatuses({ + statuses: data, + timestamp, + }) + + const statuses = processed.map(({ id }) => id) + + const repeats = processed + .filter(({ retweeted_status }) => Boolean(retweeted_status)) + .map(({ id, retweeted_status: { id: repeatedId } }) => [ + id, + repeatedId, + ]) useTimelinesStore().addStatusesToTimeline(timeline.name, argument, { - statuses: processed, + statuses, + repeats, showImmediately, older, pagination, diff --git a/src/stores/timelines.js b/src/stores/timelines.js index 26223e69d..000415b52 100644 --- a/src/stores/timelines.js +++ b/src/stores/timelines.js @@ -21,6 +21,11 @@ const emptyTl = (name, argument = null) => { // Statuses shown to user visibleStatusIds: new Set(), + // Tracked reprööts, used to avoid displaying same reprööt more than once + repeatedToRepeat: new Map(), + repeatToRepeated: new Map(), + ignoredIds: new Set(), + // Number of statuses not shown yet newStatusCount: 0, @@ -188,6 +193,7 @@ export const useTimelinesStore = defineStore('timelines', { timeline.order = [] timeline.statusIds = new Set() timeline.visibleStatusIds = new Set() + timeline.ignoredIds = new Set() timeline.newStatusCount = 0 timeline.maxId = '' timeline.minId = '' @@ -252,6 +258,7 @@ export const useTimelinesStore = defineStore('timelines', { argument, { statuses, + repeats, showImmediately = false, noIdUpdate = false, pagination = {}, @@ -261,6 +268,8 @@ export const useTimelinesStore = defineStore('timelines', { if (statuses.length === 0) return const timeline = this[timelineName] + this.populateRepeats(timeline, repeats) + // This makes sure that user timeline won't get data meant for other // user. I.e. opening different user profiles makes request which could // return data late after user already viewing different user profile @@ -287,13 +296,18 @@ export const useTimelinesStore = defineStore('timelines', { timeline.statusIds.add(statusId) if (isNew) { - if (showImmediately) { - // Add it directly to the visibleStatuses, don't change - // newStatusCount - timeline.visibleStatusIds.add(statusId) + const seenBefore = this.checkSeenBefore(timeline, statusId) + if (!seenBefore) { + if (showImmediately) { + // Add it directly to the visibleStatuses, don't change + // newStatusCount + timeline.visibleStatusIds.add(statusId) + } else { + // Just change newStatuscount + timeline.newStatusCount += 1 + } } else { - // Just change newStatuscount - timeline.newStatusCount += 1 + timeline.ignoredIds.add(statusId) } } }) @@ -301,9 +315,43 @@ export const useTimelinesStore = defineStore('timelines', { onStreamMessage(timeline, argument, event) { this.addStatusesToTimeline(timeline, argument, { statuses: event.data.map(({ id }) => id), + repeats: event.data + .filter(({ retweeted_status }) => Boolean(retweeted_status)) + .map(({ id, retweeted_status: { id: repeatedId } }) => [ + id, + repeatedId, + ]), }) }, + // Reprööt handling + populateRepeats(timeline, repeats) { + // Starting from oldest + ;[...repeats].reverse().forEach(([repeatId, repeatedId]) => { + timeline.repeatToRepeated.set(repeatId, repeatedId) + const knownRepeats = + timeline.repeatedToRepeat.get(repeatedId) ?? new Set() + knownRepeats.add(repeatId) + timeline.repeatedToRepeat.set(repeatedId, knownRepeats) + }) + }, + checkSeenBefore(timeline, statusId) { + // Check if this is a reprööt + const repeatedStatusId = timeline.repeatToRepeated.get(statusId) + // Non-reprööts are never seen before + if (!repeatedStatusId) return false + // We've seen this status already directly + if (timeline.statusIds.has(repeatedStatusId)) return true + + // Check for reprööts + const knownRepeats = timeline.repeatedToRepeat.get(repeatedStatusId) + + // If it's the only reprööt then we've never seen post before + if (knownRepeats.size === 1) return false + // If we're working on oldest known reprööt then we've never seen it before + return first(knownRepeats) !== statusId + }, + // Poll & Push onStreamConnect(timeline) { console.debug('[Timelines] Stream connected', timeline) @@ -391,7 +439,11 @@ export const useTimelinesStore = defineStore('timelines', { timeline.newStatusCount = 0 timeline.order = timeline.order.slice(0, 50) timeline.statusIds = new Set([...timeline.order]) - timeline.visibleStatusIds = new Set([...timeline.order]) + + // Reprööts CAN be present in order but some must be ignored + timeline.visibleStatusIds = new Set([ + ...timeline.order.filter((id) => !timeline.ignoredIds.has(id)), + ]) this.updateTimelineExtremes(timeline) }, syncOrder(timeline) { diff --git a/test/unit/specs/stores/timelines.spec.js b/test/unit/specs/stores/timelines.spec.js index cfc84ea3d..6b4010c71 100644 --- a/test/unit/specs/stores/timelines.spec.js +++ b/test/unit/specs/stores/timelines.spec.js @@ -122,6 +122,7 @@ describe('Timelines store', () => { store.activate('list', '1') store.addStatusesToTimeline('list', '1', { statuses, + repeats: [], pagination: { minId: '1', maxId: '3' }, }) @@ -140,10 +141,12 @@ describe('Timelines store', () => { store.activate('list', '1') store.addStatusesToTimeline('list', '1', { statuses, + repeats: [], pagination: { minId: '1', maxId: '3' }, }) store.addStatusesToTimeline('list', '1', { statuses, + repeats: [], pagination: { minId: '1', maxId: '3' }, }) @@ -162,6 +165,7 @@ describe('Timelines store', () => { store.activate('list', '1') store.addStatusesToTimeline('list', '1', { statuses, + repeats: [], showImmediately: true, pagination: { minId: '1', maxId: '3' }, }) @@ -181,6 +185,7 @@ describe('Timelines store', () => { store.activate('list', '1') store.addStatusesToTimeline('list', '1', { statuses, + repeats: [], noIdUpdate: true, pagination: { minId: '1', maxId: '3' }, }) @@ -200,6 +205,7 @@ describe('Timelines store', () => { store.activate('list', '1') store.addStatusesToTimeline('list', '2', { statuses, + repeats: [], noIdUpdate: true, pagination: { minId: '1', maxId: '3' }, }) @@ -220,10 +226,12 @@ describe('Timelines store', () => { store.activate('list', '1') store.addStatusesToTimeline('list', '1', { statuses: statuses1, + repeats: [], pagination: { minId: '3', maxId: '1' }, }) store.addStatusesToTimeline('list', '1', { statuses: statuses2, + repeats: [], pagination: { minId: '6', maxId: '4' }, }) @@ -244,10 +252,12 @@ describe('Timelines store', () => { store.activate('list', '1') store.addStatusesToTimeline('list', '1', { statuses: statuses1, + repeats: [], pagination: { minId: '6', maxId: '4' }, }) store.addStatusesToTimeline('list', '1', { statuses: statuses2, + repeats: [], pagination: { minId: '3', maxId: '1' }, older: true, }) @@ -260,6 +270,171 @@ describe('Timelines store', () => { expect(store.list).to.have.property('maxId', '6') expect(store.list).to.have.property('minId', '1') }) + + describe('repeat de-duplication', () => { + it('handles repeat de-duplication (older)', () => { + const store = useTimelinesStore() + const statuses1 = ['s7s3', 's6s3', 's5', 's4s0', 's3', 's2'] + const statuses2 = ['s1s0', 's0'] + + store.activate('list', '1') + store.addStatusesToTimeline('list', '1', { + statuses: statuses1, + repeats: [ + ['s7s3', 's2'], + ['s6s3', 's2'], + ['s4s0', 's0'], + ], + pagination: { minId: 's7s3', maxId: 's2' }, + showImmediately: true, + }) + expect(store.list.order).to.eql(statuses1) + expect(store.list.statusIds).to.eql(new Set(statuses1)) + expect(store.list.visibleStatusIds).to.eql(new Set([ + 's5', + 's4s0', + 's3', + 's2', + ])) + expect(store.list).to.have.property('maxId', 's7s3') + expect(store.list).to.have.property('minId', 's2') + + store.addStatusesToTimeline('list', '1', { + statuses: statuses2, + repeats: [ + ['s1s0', 's0'], + ], + pagination: { minId: 's1', maxId: 's0' }, + showImmediately: true, + older: true + }) + + const newOrder = [...statuses1, ...statuses2] + expect(store.list.order).to.eql(newOrder) + expect(store.list.statusIds).to.eql(new Set(newOrder)) + expect(store.list.visibleStatusIds).to.eql(new Set([ + 's5', + 's4s0', + 's3', + 's2', + 's0', + ])) + expect(store.list).to.have.property('maxId', 's7s3') + expect(store.list).to.have.property('minId', 's0') + }) + + it('handles repeat de-duplication(reverse)', () => { + const store = useTimelinesStore() + const statuses1 = ['s6s2', 's5s2', 's4', 's3s0', 's2', 's1'] + const statuses2 = ['sAs0', 's9s1', 's8', 's7s2'] + + store.activate('list', '1') + store.addStatusesToTimeline('list', '1', { + statuses: statuses1, + repeats: [ + ['s6s2', 's2'], + ['s5s2', 's2'], + ['s3s0', 's0'], + ], + pagination: { minId: 's6s2', maxId: 's1' }, + showImmediately: true, + }) + expect(store.list.order).to.eql(statuses1) + expect(store.list.statusIds).to.eql(new Set(statuses1)) + expect(store.list.visibleStatusIds).to.eql(new Set([ + 's4', + 's3s0', + 's2', + 's1', + ])) + expect(store.list).to.have.property('maxId', 's6s2') + expect(store.list).to.have.property('minId', 's1') + + store.addStatusesToTimeline('list', '1', { + statuses: statuses2, + repeats: [ + ['sAs0', 's0'], + ['s9s1', 's1'], + ['s7s2', 's2'], + ], + pagination: { minId: 'sAs0', maxId: 's7s2' }, + showImmediately: true, + }) + + const newOrder = [...statuses2, ...statuses1] + expect(store.list.order).to.eql(newOrder) + expect(store.list.statusIds).to.eql(new Set(newOrder)) + expect(store.list.visibleStatusIds).to.eql(new Set([ + 's4', + 's3s0', + 's2', + 's1', + // Newer + 's8', + ])) + expect(store.list).to.have.property('maxId', 'sAs0') + expect(store.list).to.have.property('minId', 's1') + }) + + it('showNewStatuses follows de-duplication rules', () => { + const store = useTimelinesStore() + const statuses1 = ['s6s2', 's5s2', 's4', 's3s0', 's2', 's1'] + const statuses2 = ['sAs0', 's9s1', 's8', 's7s2'] + + store.activate('list', '1') + store.addStatusesToTimeline('list', '1', { + statuses: statuses1, + repeats: [ + ['s6s2', 's2'], + ['s5s2', 's2'], + ['s3s0', 's0'], + ], + pagination: { minId: 's6s2', maxId: 's1' }, + showImmediately: true, + }) + expect(store.list.order).to.eql(statuses1) + expect(store.list.statusIds).to.eql(new Set(statuses1)) + expect(store.list.visibleStatusIds).to.eql(new Set([ + 's4', + 's3s0', + 's2', + 's1', + ])) + expect(store.list).to.have.property('maxId', 's6s2') + expect(store.list).to.have.property('minId', 's1') + + store.addStatusesToTimeline('list', '1', { + statuses: statuses2, + repeats: [ + ['sAs0', 's0'], + ['s9s1', 's1'], + ['s7s2', 's2'], + ], + pagination: { minId: 'sAs0', maxId: 's7s2' }, + }) + + const newOrder = [...statuses2, ...statuses1] + expect(store.list.order).to.eql(newOrder) + expect(store.list.statusIds).to.eql(new Set(newOrder)) + expect(store.list.newStatusCount).to.eql(1) + expect(store.list.visibleStatusIds).to.eql(new Set([ + 's4', + 's3s0', + 's2', + 's1', + ])) + + store.showNewStatuses('list') + expect(store.list.visibleStatusIds).to.eql(new Set([ + 's4', + 's3s0', + 's2', + 's1', + // Newer + 's8', + ])) + }) + }) }) describe('showNewStatuses', () => { @@ -270,6 +445,7 @@ describe('Timelines store', () => { store.activate('public') store.addStatusesToTimeline('public', undefined, { statuses, + repeats: [], pagination: { minId: '1', maxId: '3' }, }) @@ -280,6 +456,25 @@ describe('Timelines store', () => { expect(store.public.visibleStatusIds).to.eql(new Set(statuses)) expect(store.public.newStatusCount).to.equal(0) }) + + it('limits shown items to 50 for better performance', () => { + const store = useTimelinesStore() + const statuses = new Array(100).fill().map((_, index) => 's' + index) + + store.activate('public') + store.addStatusesToTimeline('public', undefined, { + statuses, + repeats: [], + pagination: { minId: '1', maxId: '3' }, + }) + + expect(store.public.statusIds).to.eql(new Set(statuses)) + expect(store.public.visibleStatusIds).to.eql(new Set()) + expect(store.public.newStatusCount).to.equal(100) + store.showNewStatuses('public') + expect(store.public.visibleStatusIds).to.eql(new Set(statuses.slice(0, 50))) + expect(store.public.newStatusCount).to.equal(0) + }) }) describe('wipeStatuses', () => { @@ -290,9 +485,11 @@ describe('Timelines store', () => { store.activate('public') store.addStatusesToTimeline('public', undefined, { statuses: ['1', '2', '3', '0'], + repeats: [], }) store.addStatusesToTimeline('friends', undefined, { statuses: ['5', '0', '9', '1'], + repeats: [], }) store.wipeStatuses(['0']) expect(store.friends.statusIds).to.not.have.members('0')