reprööt deduplication
This commit is contained in:
Henry Jameson
parent
01a99f9e8b
commit
4733eb7872
3 changed files with 275 additions and 15 deletions
|
|
@ -54,25 +54,36 @@ const timelineFetcher = (timeline, argument, credentials) => {
|
|||
|
||||
if (older && bottomedOut.value) return
|
||||
return fetchTimeline(args)
|
||||
.then(({ data: statuses, pagination, timestamp }) => {
|
||||
.then(({ data, pagination, timestamp }) => {
|
||||
// No statuses for timeline, ever.
|
||||
if (timeline.order.length === 0 && statuses.length === 0) {
|
||||
if (timeline.order.length === 0 && data.length === 0) {
|
||||
bottomedOut.value = true
|
||||
}
|
||||
if (!older && statuses.length >= 20 && numStatusesBeforeFetch > 0) {
|
||||
if (!older && data.length >= 20 && numStatusesBeforeFetch > 0) {
|
||||
useTimelinesStore().requireReload(timeline.name)
|
||||
}
|
||||
|
||||
if (older && statuses.length === 0) {
|
||||
if (older && data.length === 0) {
|
||||
bottomedOut.value = true
|
||||
}
|
||||
|
||||
const processed = useStatusesStore()
|
||||
.addNewStatuses({ statuses, timestamp })
|
||||
.map(({ id }) => id)
|
||||
const processed = useStatusesStore().addNewStatuses({
|
||||
statuses: data,
|
||||
timestamp,
|
||||
})
|
||||
|
||||
const statuses = processed.map(({ id }) => id)
|
||||
|
||||
const repeats = processed
|
||||
.filter(({ retweeted_status }) => Boolean(retweeted_status))
|
||||
.map(({ id, retweeted_status: { id: repeatedId } }) => [
|
||||
id,
|
||||
repeatedId,
|
||||
])
|
||||
|
||||
useTimelinesStore().addStatusesToTimeline(timeline.name, argument, {
|
||||
statuses: processed,
|
||||
statuses,
|
||||
repeats,
|
||||
showImmediately,
|
||||
older,
|
||||
pagination,
|
||||
|
|
|
|||
|
|
@ -21,6 +21,11 @@ const emptyTl = (name, argument = null) => {
|
|||
// Statuses shown to user
|
||||
visibleStatusIds: new Set(),
|
||||
|
||||
// Tracked reprööts, used to avoid displaying same reprööt more than once
|
||||
repeatedToRepeat: new Map(),
|
||||
repeatToRepeated: new Map(),
|
||||
ignoredIds: new Set(),
|
||||
|
||||
// Number of statuses not shown yet
|
||||
newStatusCount: 0,
|
||||
|
||||
|
|
@ -188,6 +193,7 @@ export const useTimelinesStore = defineStore('timelines', {
|
|||
timeline.order = []
|
||||
timeline.statusIds = new Set()
|
||||
timeline.visibleStatusIds = new Set()
|
||||
timeline.ignoredIds = new Set()
|
||||
timeline.newStatusCount = 0
|
||||
timeline.maxId = ''
|
||||
timeline.minId = ''
|
||||
|
|
@ -252,6 +258,7 @@ export const useTimelinesStore = defineStore('timelines', {
|
|||
argument,
|
||||
{
|
||||
statuses,
|
||||
repeats,
|
||||
showImmediately = false,
|
||||
noIdUpdate = false,
|
||||
pagination = {},
|
||||
|
|
@ -261,6 +268,8 @@ export const useTimelinesStore = defineStore('timelines', {
|
|||
if (statuses.length === 0) return
|
||||
const timeline = this[timelineName]
|
||||
|
||||
this.populateRepeats(timeline, repeats)
|
||||
|
||||
// This makes sure that user timeline won't get data meant for other
|
||||
// user. I.e. opening different user profiles makes request which could
|
||||
// return data late after user already viewing different user profile
|
||||
|
|
@ -287,13 +296,18 @@ export const useTimelinesStore = defineStore('timelines', {
|
|||
timeline.statusIds.add(statusId)
|
||||
|
||||
if (isNew) {
|
||||
if (showImmediately) {
|
||||
// Add it directly to the visibleStatuses, don't change
|
||||
// newStatusCount
|
||||
timeline.visibleStatusIds.add(statusId)
|
||||
const seenBefore = this.checkSeenBefore(timeline, statusId)
|
||||
if (!seenBefore) {
|
||||
if (showImmediately) {
|
||||
// Add it directly to the visibleStatuses, don't change
|
||||
// newStatusCount
|
||||
timeline.visibleStatusIds.add(statusId)
|
||||
} else {
|
||||
// Just change newStatuscount
|
||||
timeline.newStatusCount += 1
|
||||
}
|
||||
} else {
|
||||
// Just change newStatuscount
|
||||
timeline.newStatusCount += 1
|
||||
timeline.ignoredIds.add(statusId)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
|
@ -301,9 +315,43 @@ export const useTimelinesStore = defineStore('timelines', {
|
|||
onStreamMessage(timeline, argument, event) {
|
||||
this.addStatusesToTimeline(timeline, argument, {
|
||||
statuses: event.data.map(({ id }) => id),
|
||||
repeats: event.data
|
||||
.filter(({ retweeted_status }) => Boolean(retweeted_status))
|
||||
.map(({ id, retweeted_status: { id: repeatedId } }) => [
|
||||
id,
|
||||
repeatedId,
|
||||
]),
|
||||
})
|
||||
},
|
||||
|
||||
// Reprööt handling
|
||||
populateRepeats(timeline, repeats) {
|
||||
// Starting from oldest
|
||||
;[...repeats].reverse().forEach(([repeatId, repeatedId]) => {
|
||||
timeline.repeatToRepeated.set(repeatId, repeatedId)
|
||||
const knownRepeats =
|
||||
timeline.repeatedToRepeat.get(repeatedId) ?? new Set()
|
||||
knownRepeats.add(repeatId)
|
||||
timeline.repeatedToRepeat.set(repeatedId, knownRepeats)
|
||||
})
|
||||
},
|
||||
checkSeenBefore(timeline, statusId) {
|
||||
// Check if this is a reprööt
|
||||
const repeatedStatusId = timeline.repeatToRepeated.get(statusId)
|
||||
// Non-reprööts are never seen before
|
||||
if (!repeatedStatusId) return false
|
||||
// We've seen this status already directly
|
||||
if (timeline.statusIds.has(repeatedStatusId)) return true
|
||||
|
||||
// Check for reprööts
|
||||
const knownRepeats = timeline.repeatedToRepeat.get(repeatedStatusId)
|
||||
|
||||
// If it's the only reprööt then we've never seen post before
|
||||
if (knownRepeats.size === 1) return false
|
||||
// If we're working on oldest known reprööt then we've never seen it before
|
||||
return first(knownRepeats) !== statusId
|
||||
},
|
||||
|
||||
// Poll & Push
|
||||
onStreamConnect(timeline) {
|
||||
console.debug('[Timelines] Stream connected', timeline)
|
||||
|
|
@ -391,7 +439,11 @@ export const useTimelinesStore = defineStore('timelines', {
|
|||
timeline.newStatusCount = 0
|
||||
timeline.order = timeline.order.slice(0, 50)
|
||||
timeline.statusIds = new Set([...timeline.order])
|
||||
timeline.visibleStatusIds = new Set([...timeline.order])
|
||||
|
||||
// Reprööts CAN be present in order but some must be ignored
|
||||
timeline.visibleStatusIds = new Set([
|
||||
...timeline.order.filter((id) => !timeline.ignoredIds.has(id)),
|
||||
])
|
||||
this.updateTimelineExtremes(timeline)
|
||||
},
|
||||
syncOrder(timeline) {
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ describe('Timelines store', () => {
|
|||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses,
|
||||
repeats: [],
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
|
||||
|
|
@ -140,10 +141,12 @@ describe('Timelines store', () => {
|
|||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses,
|
||||
repeats: [],
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses,
|
||||
repeats: [],
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
|
||||
|
|
@ -162,6 +165,7 @@ describe('Timelines store', () => {
|
|||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses,
|
||||
repeats: [],
|
||||
showImmediately: true,
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
|
|
@ -181,6 +185,7 @@ describe('Timelines store', () => {
|
|||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses,
|
||||
repeats: [],
|
||||
noIdUpdate: true,
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
|
|
@ -200,6 +205,7 @@ describe('Timelines store', () => {
|
|||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '2', {
|
||||
statuses,
|
||||
repeats: [],
|
||||
noIdUpdate: true,
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
|
|
@ -220,10 +226,12 @@ describe('Timelines store', () => {
|
|||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses1,
|
||||
repeats: [],
|
||||
pagination: { minId: '3', maxId: '1' },
|
||||
})
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses2,
|
||||
repeats: [],
|
||||
pagination: { minId: '6', maxId: '4' },
|
||||
})
|
||||
|
||||
|
|
@ -244,10 +252,12 @@ describe('Timelines store', () => {
|
|||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses1,
|
||||
repeats: [],
|
||||
pagination: { minId: '6', maxId: '4' },
|
||||
})
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses2,
|
||||
repeats: [],
|
||||
pagination: { minId: '3', maxId: '1' },
|
||||
older: true,
|
||||
})
|
||||
|
|
@ -260,6 +270,171 @@ describe('Timelines store', () => {
|
|||
expect(store.list).to.have.property('maxId', '6')
|
||||
expect(store.list).to.have.property('minId', '1')
|
||||
})
|
||||
|
||||
describe('repeat de-duplication', () => {
|
||||
it('handles repeat de-duplication (older)', () => {
|
||||
const store = useTimelinesStore()
|
||||
const statuses1 = ['s7s3', 's6s3', 's5', 's4s0', 's3', 's2']
|
||||
const statuses2 = ['s1s0', 's0']
|
||||
|
||||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses1,
|
||||
repeats: [
|
||||
['s7s3', 's2'],
|
||||
['s6s3', 's2'],
|
||||
['s4s0', 's0'],
|
||||
],
|
||||
pagination: { minId: 's7s3', maxId: 's2' },
|
||||
showImmediately: true,
|
||||
})
|
||||
expect(store.list.order).to.eql(statuses1)
|
||||
expect(store.list.statusIds).to.eql(new Set(statuses1))
|
||||
expect(store.list.visibleStatusIds).to.eql(new Set([
|
||||
's5',
|
||||
's4s0',
|
||||
's3',
|
||||
's2',
|
||||
]))
|
||||
expect(store.list).to.have.property('maxId', 's7s3')
|
||||
expect(store.list).to.have.property('minId', 's2')
|
||||
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses2,
|
||||
repeats: [
|
||||
['s1s0', 's0'],
|
||||
],
|
||||
pagination: { minId: 's1', maxId: 's0' },
|
||||
showImmediately: true,
|
||||
older: true
|
||||
})
|
||||
|
||||
const newOrder = [...statuses1, ...statuses2]
|
||||
expect(store.list.order).to.eql(newOrder)
|
||||
expect(store.list.statusIds).to.eql(new Set(newOrder))
|
||||
expect(store.list.visibleStatusIds).to.eql(new Set([
|
||||
's5',
|
||||
's4s0',
|
||||
's3',
|
||||
's2',
|
||||
's0',
|
||||
]))
|
||||
expect(store.list).to.have.property('maxId', 's7s3')
|
||||
expect(store.list).to.have.property('minId', 's0')
|
||||
})
|
||||
|
||||
it('handles repeat de-duplication(reverse)', () => {
|
||||
const store = useTimelinesStore()
|
||||
const statuses1 = ['s6s2', 's5s2', 's4', 's3s0', 's2', 's1']
|
||||
const statuses2 = ['sAs0', 's9s1', 's8', 's7s2']
|
||||
|
||||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses1,
|
||||
repeats: [
|
||||
['s6s2', 's2'],
|
||||
['s5s2', 's2'],
|
||||
['s3s0', 's0'],
|
||||
],
|
||||
pagination: { minId: 's6s2', maxId: 's1' },
|
||||
showImmediately: true,
|
||||
})
|
||||
expect(store.list.order).to.eql(statuses1)
|
||||
expect(store.list.statusIds).to.eql(new Set(statuses1))
|
||||
expect(store.list.visibleStatusIds).to.eql(new Set([
|
||||
's4',
|
||||
's3s0',
|
||||
's2',
|
||||
's1',
|
||||
]))
|
||||
expect(store.list).to.have.property('maxId', 's6s2')
|
||||
expect(store.list).to.have.property('minId', 's1')
|
||||
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses2,
|
||||
repeats: [
|
||||
['sAs0', 's0'],
|
||||
['s9s1', 's1'],
|
||||
['s7s2', 's2'],
|
||||
],
|
||||
pagination: { minId: 'sAs0', maxId: 's7s2' },
|
||||
showImmediately: true,
|
||||
})
|
||||
|
||||
const newOrder = [...statuses2, ...statuses1]
|
||||
expect(store.list.order).to.eql(newOrder)
|
||||
expect(store.list.statusIds).to.eql(new Set(newOrder))
|
||||
expect(store.list.visibleStatusIds).to.eql(new Set([
|
||||
's4',
|
||||
's3s0',
|
||||
's2',
|
||||
's1',
|
||||
// Newer
|
||||
's8',
|
||||
]))
|
||||
expect(store.list).to.have.property('maxId', 'sAs0')
|
||||
expect(store.list).to.have.property('minId', 's1')
|
||||
})
|
||||
|
||||
it('showNewStatuses follows de-duplication rules', () => {
|
||||
const store = useTimelinesStore()
|
||||
const statuses1 = ['s6s2', 's5s2', 's4', 's3s0', 's2', 's1']
|
||||
const statuses2 = ['sAs0', 's9s1', 's8', 's7s2']
|
||||
|
||||
store.activate('list', '1')
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses1,
|
||||
repeats: [
|
||||
['s6s2', 's2'],
|
||||
['s5s2', 's2'],
|
||||
['s3s0', 's0'],
|
||||
],
|
||||
pagination: { minId: 's6s2', maxId: 's1' },
|
||||
showImmediately: true,
|
||||
})
|
||||
expect(store.list.order).to.eql(statuses1)
|
||||
expect(store.list.statusIds).to.eql(new Set(statuses1))
|
||||
expect(store.list.visibleStatusIds).to.eql(new Set([
|
||||
's4',
|
||||
's3s0',
|
||||
's2',
|
||||
's1',
|
||||
]))
|
||||
expect(store.list).to.have.property('maxId', 's6s2')
|
||||
expect(store.list).to.have.property('minId', 's1')
|
||||
|
||||
store.addStatusesToTimeline('list', '1', {
|
||||
statuses: statuses2,
|
||||
repeats: [
|
||||
['sAs0', 's0'],
|
||||
['s9s1', 's1'],
|
||||
['s7s2', 's2'],
|
||||
],
|
||||
pagination: { minId: 'sAs0', maxId: 's7s2' },
|
||||
})
|
||||
|
||||
const newOrder = [...statuses2, ...statuses1]
|
||||
expect(store.list.order).to.eql(newOrder)
|
||||
expect(store.list.statusIds).to.eql(new Set(newOrder))
|
||||
expect(store.list.newStatusCount).to.eql(1)
|
||||
expect(store.list.visibleStatusIds).to.eql(new Set([
|
||||
's4',
|
||||
's3s0',
|
||||
's2',
|
||||
's1',
|
||||
]))
|
||||
|
||||
store.showNewStatuses('list')
|
||||
expect(store.list.visibleStatusIds).to.eql(new Set([
|
||||
's4',
|
||||
's3s0',
|
||||
's2',
|
||||
's1',
|
||||
// Newer
|
||||
's8',
|
||||
]))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('showNewStatuses', () => {
|
||||
|
|
@ -270,6 +445,7 @@ describe('Timelines store', () => {
|
|||
store.activate('public')
|
||||
store.addStatusesToTimeline('public', undefined, {
|
||||
statuses,
|
||||
repeats: [],
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
|
||||
|
|
@ -280,6 +456,25 @@ describe('Timelines store', () => {
|
|||
expect(store.public.visibleStatusIds).to.eql(new Set(statuses))
|
||||
expect(store.public.newStatusCount).to.equal(0)
|
||||
})
|
||||
|
||||
it('limits shown items to 50 for better performance', () => {
|
||||
const store = useTimelinesStore()
|
||||
const statuses = new Array(100).fill().map((_, index) => 's' + index)
|
||||
|
||||
store.activate('public')
|
||||
store.addStatusesToTimeline('public', undefined, {
|
||||
statuses,
|
||||
repeats: [],
|
||||
pagination: { minId: '1', maxId: '3' },
|
||||
})
|
||||
|
||||
expect(store.public.statusIds).to.eql(new Set(statuses))
|
||||
expect(store.public.visibleStatusIds).to.eql(new Set())
|
||||
expect(store.public.newStatusCount).to.equal(100)
|
||||
store.showNewStatuses('public')
|
||||
expect(store.public.visibleStatusIds).to.eql(new Set(statuses.slice(0, 50)))
|
||||
expect(store.public.newStatusCount).to.equal(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('wipeStatuses', () => {
|
||||
|
|
@ -290,9 +485,11 @@ describe('Timelines store', () => {
|
|||
store.activate('public')
|
||||
store.addStatusesToTimeline('public', undefined, {
|
||||
statuses: ['1', '2', '3', '0'],
|
||||
repeats: [],
|
||||
})
|
||||
store.addStatusesToTimeline('friends', undefined, {
|
||||
statuses: ['5', '0', '9', '1'],
|
||||
repeats: [],
|
||||
})
|
||||
store.wipeStatuses(['0'])
|
||||
expect(store.friends.statusIds).to.not.have.members('0')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue