feat(crm): détection doublons fusion tri et abcédaire

EBO <eric.bouhana@softalys.com> committé le 2026-07-22 15:29

b4cb8c5336ec8643c7a3afd11296eeff76a10d2f

1 parent(s)

9 fichiers modifiés +496 -0
A android/app/src/main/java/fr/ebii/card2vcf/crm/AlphabetIndex.kt
+45 -0
@@ -0,0 +1,45 @@
1 +package fr.ebii.card2vcf.crm
2 +
3 +import fr.ebii.card2vcf.data.CrmContactEntity
4 +
5 +object AlphabetIndex {
6 + fun sortKey(c: CrmContactEntity, sort: ContactSort): String = when (sort) {
7 + ContactSort.LAST_NAME -> c.lastName ?: c.fullName ?: ""
8 + ContactSort.FIRST_NAME -> c.firstName ?: c.fullName ?: ""
9 + ContactSort.COMPANY -> c.company ?: ""
10 + ContactSort.CREATED_AT -> c.createdAt.toString().padStart(20, '0')
11 + }
12 +
13 + fun sectionLetter(key: String): Char {
14 + val c = key.trim().firstOrNull()?.uppercaseChar()
15 + return if (c != null && c in 'A'..'Z') c else '#'
16 + }
17 +
18 + /**
19 + * Alphabetical sorts: ordered A–Z then `#`, contacts sorted within each section.
20 + * [ContactSort.CREATED_AT]: single flat section (chrono desc), no A–Z rail sections.
21 + */
22 + fun group(
23 + contacts: List<CrmContactEntity>,
24 + sort: ContactSort,
25 + ): LinkedHashMap<Char, List<CrmContactEntity>> {
26 + if (sort == ContactSort.CREATED_AT) {
27 + val sorted = contacts.sortedByDescending { it.createdAt }
28 + return linkedMapOf('#' to sorted)
29 + }
30 + val sorted = contacts.sortedWith(
31 + compareBy(String.CASE_INSENSITIVE_ORDER) { sortKey(it, sort) },
32 + )
33 + val buckets = LinkedHashMap<Char, MutableList<CrmContactEntity>>()
34 + for (c in sorted) {
35 + val letter = sectionLetter(sortKey(c, sort))
36 + buckets.getOrPut(letter) { mutableListOf() }.add(c)
37 + }
38 + val out = LinkedHashMap<Char, List<CrmContactEntity>>()
39 + for ((k, v) in buckets) {
40 + if (k != '#') out[k] = v
41 + }
42 + buckets['#']?.let { out['#'] = it }
43 + return out
44 + }
45 +}
A android/app/src/main/java/fr/ebii/card2vcf/crm/ContactMerge.kt
+55 -0
@@ -0,0 +1,55 @@
1 +package fr.ebii.card2vcf.crm
2 +
3 +import fr.ebii.card2vcf.data.CrmContactEntity
4 +
5 +data class MergeFieldChoices(
6 + val fullNameFromId: Long,
7 + val firstNameFromId: Long,
8 + val lastNameFromId: Long,
9 + val companyFromId: Long,
10 + val jobTitleFromId: Long,
11 + val websiteFromId: Long,
12 + val addressFromId: Long,
13 + val notesMode: NotesMode,
14 + val cardImageFromId: Long?,
15 + val profileImageFromId: Long?,
16 + val keepId: Long,
17 +)
18 +
19 +enum class NotesMode { KEEP_PRIMARY, CONCAT }
20 +
21 +object ContactMerge {
22 + fun merge(
23 + sources: Map<Long, CrmContactEntity>,
24 + choices: MergeFieldChoices,
25 + now: Long = System.currentTimeMillis(),
26 + ): CrmContactEntity {
27 + fun scalar(picker: Long, sel: (CrmContactEntity) -> String?) =
28 + sources.getValue(picker).let(sel)
29 + val keep = sources.getValue(choices.keepId)
30 + val phones = sources.values.flatMap { it.phones }.distinct()
31 + val emails = sources.values.flatMap { it.emails }.distinct()
32 + val notes = when (choices.notesMode) {
33 + NotesMode.KEEP_PRIMARY -> keep.notes
34 + NotesMode.CONCAT -> sources.values.mapNotNull { it.notes?.takeIf { n -> n.isNotBlank() } }
35 + .distinct()
36 + .joinToString("\n---\n")
37 + .ifBlank { null }
38 + }
39 + return keep.copy(
40 + fullName = scalar(choices.fullNameFromId) { it.fullName },
41 + firstName = scalar(choices.firstNameFromId) { it.firstName },
42 + lastName = scalar(choices.lastNameFromId) { it.lastName },
43 + company = scalar(choices.companyFromId) { it.company },
44 + jobTitle = scalar(choices.jobTitleFromId) { it.jobTitle },
45 + website = scalar(choices.websiteFromId) { it.website },
46 + address = scalar(choices.addressFromId) { it.address },
47 + notes = notes,
48 + phones = phones,
49 + emails = emails,
50 + phonesText = phones.joinToString(" "),
51 + emailsText = emails.joinToString(" "),
52 + updatedAt = now,
53 + )
54 + }
55 +}
A android/app/src/main/java/fr/ebii/card2vcf/crm/ContactSort.kt
+3 -0
@@ -0,0 +1,3 @@
1 +package fr.ebii.card2vcf.crm
2 +
3 +enum class ContactSort { LAST_NAME, FIRST_NAME, COMPANY, CREATED_AT }
A android/app/src/main/java/fr/ebii/card2vcf/crm/DuplicateDetector.kt
+58 -0
@@ -0,0 +1,58 @@
1 +package fr.ebii.card2vcf.crm
2 +
3 +import fr.ebii.card2vcf.data.CrmContactEntity
4 +import fr.ebii.card2vcf.data.DuplicateExclusionEntity
5 +
6 +object DuplicateDetector {
7 + fun clusters(
8 + contacts: List<CrmContactEntity>,
9 + exclusions: List<DuplicateExclusionEntity>,
10 + ): Map<Long, Set<Long>> {
11 + val parent = contacts.associate { it.id to it.id }.toMutableMap()
12 + fun find(x: Long): Long {
13 + var p = x
14 + while (parent[p] != p) p = parent[p]!!
15 + return p
16 + }
17 + fun union(a: Long, b: Long) {
18 + val ra = find(a)
19 + val rb = find(b)
20 + if (ra != rb) parent[ra] = rb
21 + }
22 + val excluded = exclusions.map { it.idA to it.idB }.toSet()
23 + fun excludedPair(a: Long, b: Long): Boolean {
24 + val lo = minOf(a, b)
25 + val hi = maxOf(a, b)
26 + return (lo to hi) in excluded
27 + }
28 + val byEmail = mutableMapOf<String, MutableList<Long>>()
29 + val byPhone = mutableMapOf<String, MutableList<Long>>()
30 + for (c in contacts) {
31 + c.emails.map(ContactNormalizer::email).filter { it.isNotEmpty() }
32 + .forEach { byEmail.getOrPut(it) { mutableListOf() }.add(c.id) }
33 + c.phones.map(ContactNormalizer::phone).filter { it.isNotEmpty() }
34 + .forEach { byPhone.getOrPut(it) { mutableListOf() }.add(c.id) }
35 + }
36 + fun linkGroups(groups: Collection<List<Long>>) {
37 + for (g in groups) {
38 + for (i in 1 until g.size) {
39 + if (!excludedPair(g[0], g[i])) union(g[0], g[i])
40 + }
41 + }
42 + }
43 + linkGroups(byEmail.values)
44 + linkGroups(byPhone.values)
45 + val rootToMembers = mutableMapOf<Long, MutableSet<Long>>()
46 + for (c in contacts) {
47 + rootToMembers.getOrPut(find(c.id)) { mutableSetOf() }.add(c.id)
48 + }
49 + val idToCluster = mutableMapOf<Long, Set<Long>>()
50 + for (members in rootToMembers.values) {
51 + for (id in members) idToCluster[id] = members
52 + }
53 + return idToCluster
54 + }
55 +
56 + fun duplicateCount(contactId: Long, clusters: Map<Long, Set<Long>>): Int =
57 + ((clusters[contactId]?.size ?: 1) - 1).coerceAtLeast(0)
58 +}
M android/app/src/main/java/fr/ebii/card2vcf/data/ContactRepository.kt
+31 -0
@@ -2,7 +2,9 @@ package fr.ebii.card2vcf.data
2 2
3 3 import android.graphics.Bitmap
4 4 import fr.ebii.card2vcf.contact.ContactCard
5 +import fr.ebii.card2vcf.crm.ContactMerge
5 6 import fr.ebii.card2vcf.crm.ContactNormalizer
7 +import fr.ebii.card2vcf.crm.MergeFieldChoices
6 8 import kotlinx.coroutines.flow.Flow
7 9
8 10 data class VcfImportResult(
@@ -81,6 +83,35 @@ class ContactRepository(
81 83 images.deleteAll(id)
82 84 }
83 85
86 + suspend fun markDifferent(a: Long, b: Long) {
87 + val lo = minOf(a, b)
88 + val hi = maxOf(a, b)
89 + dao.insertExclusion(DuplicateExclusionEntity(idA = lo, idB = hi))
90 + }
91 +
92 + suspend fun mergeContacts(
93 + sources: List<CrmContactEntity>,
94 + choices: MergeFieldChoices,
95 + ) {
96 + val map = sources.associateBy { it.id }
97 + var merged = ContactMerge.merge(map, choices)
98 + val keepId = choices.keepId
99 + choices.cardImageFromId?.let { srcId ->
100 + val path = map[srcId]?.cardImagePath
101 + val copied = images.copyImage(path, keepId, "card.jpg")
102 + if (copied != null) merged = merged.copy(cardImagePath = copied)
103 + }
104 + choices.profileImageFromId?.let { srcId ->
105 + val path = map[srcId]?.profileImagePath
106 + val copied = images.copyImage(path, keepId, "profile.jpg")
107 + if (copied != null) merged = merged.copy(profileImagePath = copied)
108 + }
109 + dao.update(merged)
110 + for (s in sources) {
111 + if (s.id != keepId) delete(s.id)
112 + }
113 + }
114 +
84 115 suspend fun findMatchingId(card: ContactCard, among: List<CrmContactEntity>): Long? {
85 116 val emails = card.emails.map(ContactNormalizer::email).filter { it.isNotEmpty() }.toSet()
86 117 val phones = card.phones.map(ContactNormalizer::phone).filter { it.isNotEmpty() }.toSet()
A android/app/src/test/java/fr/ebii/card2vcf/crm/AlphabetIndexTest.kt
+72 -0
@@ -0,0 +1,72 @@
1 +package fr.ebii.card2vcf.crm
2 +
3 +import fr.ebii.card2vcf.data.CrmContactEntity
4 +import kotlin.test.Test
5 +import kotlin.test.assertEquals
6 +
7 +class AlphabetIndexTest {
8 + private fun contact(
9 + id: Long,
10 + lastName: String? = null,
11 + firstName: String? = null,
12 + fullName: String? = null,
13 + company: String? = null,
14 + createdAt: Long = 0L,
15 + ) = CrmContactEntity(
16 + id = id,
17 + lastName = lastName,
18 + firstName = firstName,
19 + fullName = fullName,
20 + company = company,
21 + createdAt = createdAt,
22 + )
23 +
24 + @Test
25 + fun sectionLetterAzAndHash() {
26 + assertEquals('A', AlphabetIndex.sectionLetter("alice"))
27 + assertEquals('Z', AlphabetIndex.sectionLetter(" zebra"))
28 + assertEquals('#', AlphabetIndex.sectionLetter(""))
29 + assertEquals('#', AlphabetIndex.sectionLetter(" "))
30 + assertEquals('#', AlphabetIndex.sectionLetter("Émile"))
31 + assertEquals('#', AlphabetIndex.sectionLetter("9ops"))
32 + }
33 +
34 + @Test
35 + fun sortKeyFallbacks() {
36 + val withLast = contact(1, lastName = "Dupont", fullName = "Jean Dupont")
37 + val onlyFull = contact(2, fullName = "Solo")
38 + assertEquals("Dupont", AlphabetIndex.sortKey(withLast, ContactSort.LAST_NAME))
39 + assertEquals("Solo", AlphabetIndex.sortKey(onlyFull, ContactSort.LAST_NAME))
40 + assertEquals("Solo", AlphabetIndex.sortKey(onlyFull, ContactSort.FIRST_NAME))
41 + assertEquals("", AlphabetIndex.sortKey(contact(3), ContactSort.COMPANY))
42 + assertEquals("00000000000000000042", AlphabetIndex.sortKey(contact(4, createdAt = 42), ContactSort.CREATED_AT))
43 + }
44 +
45 + @Test
46 + fun groupByLastNameSections() {
47 + val contacts = listOf(
48 + contact(1, lastName = "Bernard"),
49 + contact(2, lastName = "Adams"),
50 + contact(3, lastName = "Dupont"),
51 + contact(4, lastName = "123"),
52 + )
53 + val grouped = AlphabetIndex.group(contacts, ContactSort.LAST_NAME)
54 + assertEquals(listOf('A', 'B', 'D', '#'), grouped.keys.toList())
55 + assertEquals(listOf(2L), grouped['A']!!.map { it.id })
56 + assertEquals(listOf(1L), grouped['B']!!.map { it.id })
57 + assertEquals(listOf(3L), grouped['D']!!.map { it.id })
58 + assertEquals(listOf(4L), grouped['#']!!.map { it.id })
59 + }
60 +
61 + @Test
62 + fun createdAtFlatChronoDescNoAzSections() {
63 + val contacts = listOf(
64 + contact(1, lastName = "A", createdAt = 10),
65 + contact(2, lastName = "B", createdAt = 30),
66 + contact(3, lastName = "C", createdAt = 20),
67 + )
68 + val grouped = AlphabetIndex.group(contacts, ContactSort.CREATED_AT)
69 + assertEquals(1, grouped.size)
70 + assertEquals(listOf(2L, 3L, 1L), grouped.values.single().map { it.id })
71 + }
72 +}
A android/app/src/test/java/fr/ebii/card2vcf/crm/ContactMergeTest.kt
+137 -0
@@ -0,0 +1,137 @@
1 +package fr.ebii.card2vcf.crm
2 +
3 +import fr.ebii.card2vcf.data.CrmContactEntity
4 +import kotlin.test.Test
5 +import kotlin.test.assertEquals
6 +import kotlin.test.assertNull
7 +
8 +class ContactMergeTest {
9 + private fun contact(
10 + id: Long,
11 + fullName: String? = null,
12 + firstName: String? = null,
13 + lastName: String? = null,
14 + company: String? = null,
15 + jobTitle: String? = null,
16 + website: String? = null,
17 + address: String? = null,
18 + notes: String? = null,
19 + phones: List<String> = emptyList(),
20 + emails: List<String> = emptyList(),
21 + ) = CrmContactEntity(
22 + id = id,
23 + fullName = fullName,
24 + firstName = firstName,
25 + lastName = lastName,
26 + company = company,
27 + jobTitle = jobTitle,
28 + website = website,
29 + address = address,
30 + notes = notes,
31 + phones = phones,
32 + emails = emails,
33 + phonesText = phones.joinToString(" "),
34 + emailsText = emails.joinToString(" "),
35 + createdAt = id,
36 + updatedAt = id,
37 + )
38 +
39 + private fun choices(
40 + keepId: Long,
41 + notesMode: NotesMode = NotesMode.KEEP_PRIMARY,
42 + fullNameFromId: Long = keepId,
43 + firstNameFromId: Long = keepId,
44 + lastNameFromId: Long = keepId,
45 + companyFromId: Long = keepId,
46 + jobTitleFromId: Long = keepId,
47 + websiteFromId: Long = keepId,
48 + addressFromId: Long = keepId,
49 + ) = MergeFieldChoices(
50 + fullNameFromId = fullNameFromId,
51 + firstNameFromId = firstNameFromId,
52 + lastNameFromId = lastNameFromId,
53 + companyFromId = companyFromId,
54 + jobTitleFromId = jobTitleFromId,
55 + websiteFromId = websiteFromId,
56 + addressFromId = addressFromId,
57 + notesMode = notesMode,
58 + cardImageFromId = null,
59 + profileImageFromId = null,
60 + keepId = keepId,
61 + )
62 +
63 + @Test
64 + fun picksScalarsAndUnionsLists() {
65 + val a = contact(
66 + 1,
67 + fullName = "A",
68 + firstName = "Ada",
69 + company = "CoA",
70 + notes = "note A",
71 + phones = listOf("111"),
72 + emails = listOf("a@ex.com"),
73 + )
74 + val b = contact(
75 + 2,
76 + fullName = "B",
77 + lastName = "Lovelace",
78 + jobTitle = "Eng",
79 + website = "https://b",
80 + address = "Paris",
81 + notes = "note B",
82 + phones = listOf("111", "222"),
83 + emails = listOf("b@ex.com"),
84 + )
85 + val merged = ContactMerge.merge(
86 + mapOf(1L to a, 2L to b),
87 + choices(
88 + keepId = 1,
89 + fullNameFromId = 2,
90 + lastNameFromId = 2,
91 + jobTitleFromId = 2,
92 + websiteFromId = 2,
93 + addressFromId = 2,
94 + ),
95 + now = 99L,
96 + )
97 + assertEquals(1L, merged.id)
98 + assertEquals("B", merged.fullName)
99 + assertEquals("Ada", merged.firstName)
100 + assertEquals("Lovelace", merged.lastName)
101 + assertEquals("CoA", merged.company)
102 + assertEquals("Eng", merged.jobTitle)
103 + assertEquals("https://b", merged.website)
104 + assertEquals("Paris", merged.address)
105 + assertEquals("note A", merged.notes)
106 + assertEquals(listOf("111", "222"), merged.phones)
107 + assertEquals(listOf("a@ex.com", "b@ex.com"), merged.emails)
108 + assertEquals("111 222", merged.phonesText)
109 + assertEquals("a@ex.com b@ex.com", merged.emailsText)
110 + assertEquals(99L, merged.updatedAt)
111 + assertEquals(1L, merged.createdAt)
112 + }
113 +
114 + @Test
115 + fun concatNotes() {
116 + val a = contact(1, notes = "alpha")
117 + val b = contact(2, notes = "beta")
118 + val merged = ContactMerge.merge(
119 + mapOf(1L to a, 2L to b),
120 + choices(keepId = 1, notesMode = NotesMode.CONCAT),
121 + now = 1L,
122 + )
123 + assertEquals("alpha\n---\nbeta", merged.notes)
124 + }
125 +
126 + @Test
127 + fun concatNotesBlankBecomesNull() {
128 + val a = contact(1, notes = " ")
129 + val b = contact(2, notes = null)
130 + val merged = ContactMerge.merge(
131 + mapOf(1L to a, 2L to b),
132 + choices(keepId = 1, notesMode = NotesMode.CONCAT),
133 + now = 1L,
134 + )
135 + assertNull(merged.notes)
136 + }
137 +}
A android/app/src/test/java/fr/ebii/card2vcf/crm/DuplicateDetectorTest.kt
+48 -0
@@ -0,0 +1,48 @@
1 +package fr.ebii.card2vcf.crm
2 +
3 +import fr.ebii.card2vcf.data.CrmContactEntity
4 +import fr.ebii.card2vcf.data.DuplicateExclusionEntity
5 +import kotlin.test.Test
6 +import kotlin.test.assertEquals
7 +import kotlin.test.assertTrue
8 +
9 +class DuplicateDetectorTest {
10 + private fun contact(
11 + id: Long,
12 + emails: List<String> = emptyList(),
13 + phones: List<String> = emptyList(),
14 + ) = CrmContactEntity(id = id, emails = emails, phones = phones)
15 +
16 + @Test
17 + fun sameEmailSameCluster() {
18 + val a = contact(1, emails = listOf("Ada@Example.COM"))
19 + val b = contact(2, emails = listOf(" ada@example.com "))
20 + val c = contact(3, emails = listOf("other@ex.com"))
21 + val clusters = DuplicateDetector.clusters(listOf(a, b, c), emptyList())
22 + assertEquals(setOf(1L, 2L), clusters[1])
23 + assertEquals(setOf(1L, 2L), clusters[2])
24 + assertEquals(setOf(3L), clusters[3])
25 + assertEquals(1, DuplicateDetector.duplicateCount(1, clusters))
26 + assertEquals(0, DuplicateDetector.duplicateCount(3, clusters))
27 + }
28 +
29 + @Test
30 + fun exclusionSeparatesCluster() {
31 + val a = contact(1, emails = listOf("same@ex.com"))
32 + val b = contact(2, emails = listOf("same@ex.com"))
33 + val exclusions = listOf(DuplicateExclusionEntity(idA = 1, idB = 2))
34 + val clusters = DuplicateDetector.clusters(listOf(a, b), exclusions)
35 + assertEquals(setOf(1L), clusters[1])
36 + assertEquals(setOf(2L), clusters[2])
37 + assertEquals(0, DuplicateDetector.duplicateCount(1, clusters))
38 + }
39 +
40 + @Test
41 + fun normalizedPhoneMatch() {
42 + val a = contact(10, phones = listOf("+33 6 12-34"))
43 + val b = contact(20, phones = listOf("3361234"))
44 + val clusters = DuplicateDetector.clusters(listOf(a, b), emptyList())
45 + assertEquals(setOf(10L, 20L), clusters[10])
46 + assertTrue(DuplicateDetector.duplicateCount(10, clusters) == 1)
47 + }
48 +}
M android/app/src/test/java/fr/ebii/card2vcf/data/ContactRepositoryTest.kt
+47 -0
@@ -4,6 +4,8 @@ import android.content.Context
4 4 import androidx.room.Room
5 5 import androidx.test.core.app.ApplicationProvider
6 6 import fr.ebii.card2vcf.contact.ContactCard
7 +import fr.ebii.card2vcf.crm.MergeFieldChoices
8 +import fr.ebii.card2vcf.crm.NotesMode
7 9 import kotlinx.coroutines.runBlocking
8 10 import org.junit.After
9 11 import org.junit.Assert.assertEquals
@@ -78,4 +80,49 @@ class ContactRepositoryTest {
78 80 assertNull(dao.getById(id))
79 81 assertTrue(dao.getAll().isEmpty())
80 82 }
83 +
84 + @Test
85 + fun markDifferentStoresOrderedExclusion() = runBlocking {
86 + repo.markDifferent(5, 2)
87 + val ex = dao.getExclusions().single()
88 + assertEquals(2L, ex.idA)
89 + assertEquals(5L, ex.idB)
90 + }
91 +
92 + @Test
93 + fun mergeContactsKeepsOneAndDeletesOthers() = runBlocking {
94 + val idA = repo.insertFromCard(
95 + ContactCard(fullName = "A", emails = listOf("a@ex.com"), phones = listOf("111")),
96 + null,
97 + null,
98 + )
99 + val idB = repo.insertFromCard(
100 + ContactCard(fullName = "B", emails = listOf("b@ex.com"), phones = listOf("222")),
101 + null,
102 + null,
103 + )
104 + val a = dao.getById(idA)!!
105 + val b = dao.getById(idB)!!
106 + repo.mergeContacts(
107 + listOf(a, b),
108 + MergeFieldChoices(
109 + fullNameFromId = idB,
110 + firstNameFromId = idA,
111 + lastNameFromId = idA,
112 + companyFromId = idA,
113 + jobTitleFromId = idA,
114 + websiteFromId = idA,
115 + addressFromId = idA,
116 + notesMode = NotesMode.KEEP_PRIMARY,
117 + cardImageFromId = null,
118 + profileImageFromId = null,
119 + keepId = idA,
120 + ),
121 + )
122 + assertNull(dao.getById(idB))
123 + val kept = dao.getById(idA)!!
124 + assertEquals("B", kept.fullName)
125 + assertEquals(listOf("111", "222"), kept.phones)
126 + assertEquals(listOf("a@ex.com", "b@ex.com"), kept.emails)
127 + }
81 128 }