added mark deleeted for m18id no more exist in the PO/DO line

src/main/java/com/ffii/fpsms/m18/service/M18DeliveryOrderService.kt

@@ -275,8 +275,8 @@ open class M18DeliveryOrderService(
                             }
 
                             // delivery_order_line + m18_data_log
-                            // TODO: check deleted po line?
                             if (pot != null) {
+                                val m18LineIds = pot.map { it.id }.toSet()
                                 // Loop for Delivery Order Lines (pot)
                                 pot.forEach { line ->
 
@@ -393,6 +393,25 @@ open class M18DeliveryOrderService(
 //                                        logger.error("${doLineRefType}: M18 Data Log Updated! Please see the error. ID: ${saveM18DeliveryOrderLineLog.id}")
                                     }
                                 }
+
+                                if (deliveryOrderId != null) {
+                                    val markedDeleted = deliveryOrderLineService.markDeletedLinesMissingFromM18(
+                                        deliveryOrderId = deliveryOrderId,
+                                        existingM18LineIds = m18LineIds,
+                                        m18RefType = doLineRefType
+                                    )
+                                    if (markedDeleted > 0) {
+                                        logger.info("${doLineRefType}: Marked ${markedDeleted} stale line(s) as deleted for deliveryOrderId=${deliveryOrderId}")
+                                    }
+                                    val dupesRemoved = deliveryOrderLineService.markDuplicateDeliveryOrderLinesForSameM18LineId(
+                                        deliveryOrderId = deliveryOrderId,
+                                        existingM18LineIds = m18LineIds,
+                                        m18RefType = doLineRefType
+                                    )
+                                    if (dupesRemoved > 0) {
+                                        logger.info("${doLineRefType}: Marked ${dupesRemoved} duplicate line(s) as deleted (same M18 line id) for deliveryOrderId=${deliveryOrderId}")
+                                    }
+                                }
                             } else {
                                 // pot
 //                                logger.error("${doLineRefType}: Saving Failure!")

src/main/java/com/ffii/fpsms/m18/service/M18PurchaseOrderService.kt

@@ -435,11 +435,17 @@ open class M18PurchaseOrderService(
                                     logger.info("${poLineRefType}: Item ID: ${itemId} | M18 Item ID: ${line.proId}")
 
                                     try {
-                                        // Find the purchase_order_line if exist
+                                        // Find the purchase_order_line if exist (stable key: PO + M18 line id)
 //                                        logger.info("${poLineRefType}: Finding exising purchase order line...")
-                                        val existingPurchaseOrderLine = latestPurchaseOrderLineLog?.id?.let {
-                                            purchaseOrderLineService.findPurchaseOrderLineByM18Id(it)
-                                        }
+                                        val existingPurchaseOrderLine =
+                                            purchaseOrderId?.let { pid ->
+                                                purchaseOrderLineService.findPurchaseOrderLineByPurchaseOrderAndM18LineId(
+                                                    pid,
+                                                    line.id
+                                                )
+                                            } ?: latestPurchaseOrderLineLog?.id?.let {
+                                                purchaseOrderLineService.findPurchaseOrderLineByM18Id(it)
+                                            }
 //                                        logger.info("${poLineRefType}: Exising purchase order line ID: ${existingPurchaseOrderLine?.id}")
 
                                         // Save to purchase_order_line table
@@ -517,6 +523,15 @@ open class M18PurchaseOrderService(
                                     if (markedDeleted > 0) {
                                         logger.info("${poLineRefType}: Marked $markedDeleted line(s) as deleted (not in M18). PO ID: $purchaseOrderId | M18 PO ID: ${purchaseOrder.id}")
                                     }
+                                    val (dupesRemoved, dupeItemIds) =
+                                        purchaseOrderLineService.markDuplicatePurchaseOrderLinesForSameM18LineId(
+                                            purchaseOrderId,
+                                            m18LineIds
+                                        )
+                                    affectedItemIds.addAll(dupeItemIds)
+                                    if (dupesRemoved > 0) {
+                                        logger.info("${poLineRefType}: Marked $dupesRemoved duplicate line(s) as deleted (same M18 line id). PO ID: $purchaseOrderId | M18 PO ID: ${purchaseOrder.id}")
+                                    }
                                 }
                             } else {
                                 // pot

src/main/java/com/ffii/fpsms/modules/deliveryOrder/entity/DeliveryOrderLineRepository.kt

@@ -9,6 +9,17 @@ import java.io.Serializable
 interface DeliveryOrderLineRepository : AbstractRepository<DeliveryOrderLine, Long> {
     fun findByM18DataLogIdAndDeletedIsFalse(m18datalogId: Serializable): DeliveryOrderLine?
 
+    @Query(
+        "SELECT dol FROM DeliveryOrderLine dol " +
+            "WHERE dol.deleted = false " +
+            "AND dol.deliveryOrder.id = :deliveryOrderId " +
+            "AND dol.m18DataLog.refType = :refType"
+    )
+    fun findAllByDeliveryOrderIdAndM18RefTypeAndDeletedIsFalse(
+        deliveryOrderId: Long,
+        refType: String
+    ): List<DeliveryOrderLine>
+
     @Query(
         "SELECT dol FROM DeliveryOrderLine dol " +
             "WHERE dol.deleted = false AND dol.item IS NOT NULL AND dol.item.isFee = true"

src/main/java/com/ffii/fpsms/modules/deliveryOrder/service/DeliveryOrderLineService.kt

@@ -77,4 +77,63 @@ open class DeliveryOrderLineService(
         }
         return feeLines.size
     }
+
+    open fun markDeletedLinesMissingFromM18(
+        deliveryOrderId: Long,
+        existingM18LineIds: Set<Long>,
+        m18RefType: String
+    ): Int {
+        val localLines = deliveryOrderLineRepository.findAllByDeliveryOrderIdAndM18RefTypeAndDeletedIsFalse(
+            deliveryOrderId = deliveryOrderId,
+            refType = m18RefType
+        )
+
+        val linesToDelete = localLines.filter { line ->
+            val m18LineId = line.m18DataLog?.m18Id
+            m18LineId == null || !existingM18LineIds.contains(m18LineId)
+        }
+
+        linesToDelete.forEach { it.deleted = true }
+        if (linesToDelete.isNotEmpty()) {
+            deliveryOrderLineRepository.saveAll(linesToDelete)
+            deliveryOrderLineRepository.flush()
+        }
+        return linesToDelete.size
+    }
+
+    /**
+     * After M18 sync, multiple [DeliveryOrderLine] rows can exist for the same M18 line id: each sync
+     * inserts a new m18_data_log row, and lookup by latest log id may not attach to the row still
+     * pointing at an older log. Keep the newest row (highest id) per m18DataLog.m18Id and mark others deleted.
+     */
+    open fun markDuplicateDeliveryOrderLinesForSameM18LineId(
+        deliveryOrderId: Long,
+        existingM18LineIds: Set<Long>,
+        m18RefType: String
+    ): Int {
+        val localLines = deliveryOrderLineRepository.findAllByDeliveryOrderIdAndM18RefTypeAndDeletedIsFalse(
+            deliveryOrderId = deliveryOrderId,
+            refType = m18RefType
+        )
+        val byM18LineId = localLines
+            .mapNotNull { line ->
+                val mid = line.m18DataLog?.m18Id ?: return@mapNotNull null
+                if (mid !in existingM18LineIds) return@mapNotNull null
+                mid to line
+            }
+            .groupBy({ it.first }, { it.second })
+
+        val toSoftDelete = mutableListOf<DeliveryOrderLine>()
+        for ((_, lines) in byM18LineId) {
+            if (lines.size <= 1) continue
+            val sorted = lines.sortedByDescending { it.id ?: 0L }
+            toSoftDelete.addAll(sorted.drop(1))
+        }
+        toSoftDelete.forEach { it.deleted = true }
+        if (toSoftDelete.isNotEmpty()) {
+            deliveryOrderLineRepository.saveAll(toSoftDelete)
+            deliveryOrderLineRepository.flush()
+        }
+        return toSoftDelete.size
+    }
 }

src/main/java/com/ffii/fpsms/modules/purchaseOrder/entity/PurchaseOrderLineRepository.kt

@@ -4,6 +4,7 @@ import com.ffii.core.support.AbstractRepository
 import com.ffii.fpsms.modules.purchaseOrder.entity.projections.PurchaseOrderLineInfo
 import com.ffii.fpsms.modules.purchaseOrder.enums.PurchaseOrderLineStatus
 import org.springframework.data.jpa.repository.Query
+import org.springframework.data.repository.query.Param
 import org.springframework.stereotype.Repository
 import java.io.Serializable
 
@@ -17,6 +18,19 @@ interface PurchaseOrderLineRepository : AbstractRepository<PurchaseOrderLine, Lo
 //    fun findAllByPurchaseOrderIdAndDeletedIsFalse(purchaseOrderId: Long): List<PurchaseOrderLine>
     //    fun find
 
+    /**
+     * M18 sync: resolve the local line by PO + M18 line id (m18_data_log.m18Id), not by latest log row id.
+     * Newest id first so we attach the new sync to the canonical row when duplicates exist.
+     */
+    @Query(
+        "SELECT pol FROM PurchaseOrderLine pol WHERE pol.deleted = false " +
+            "AND pol.purchaseOrder.id = :purchaseOrderId AND pol.m18DataLog.m18Id = :m18LineId ORDER BY pol.id DESC"
+    )
+    fun findAllByPurchaseOrderIdAndM18LineIdOrderByIdDesc(
+        @Param("purchaseOrderId") purchaseOrderId: Long,
+        @Param("m18LineId") m18LineId: Long
+    ): List<PurchaseOrderLine>
+
     @Query("SELECT pol FROM PurchaseOrderLine pol WHERE pol.deleted = false AND pol.item IS NOT NULL AND pol.item.isFee = true")
     fun findAllByDeletedIsFalseAndItemIsFeeTrue(): List<PurchaseOrderLine>
 }

src/main/java/com/ffii/fpsms/modules/purchaseOrder/service/PurchaseOrderLineService.kt

@@ -34,6 +34,18 @@ open class PurchaseOrderLineService(
         return purchaseOrderLineRepository.findByM18DataLogIdAndDeletedIsFalse(m18DataLogId)
     }
 
+    /**
+     * Resolve local PO line for M18 sync by [purchaseOrderId] + M18 line id ([m18LineId] = pot line id).
+     * Prefer this over [findPurchaseOrderLineByM18Id] so repeated syncs update the same row instead of inserting duplicates.
+     */
+    open fun findPurchaseOrderLineByPurchaseOrderAndM18LineId(
+        purchaseOrderId: Long,
+        m18LineId: Long
+    ): PurchaseOrderLine? =
+        purchaseOrderLineRepository
+            .findAllByPurchaseOrderIdAndM18LineIdOrderByIdDesc(purchaseOrderId, m18LineId)
+            .firstOrNull()
+
     /**
      * Mark as deleted any local PO lines for this PO that were synced from M18 but whose M18 line id
      * is not in the given set (i.e. the line was deleted in M18).
@@ -56,6 +68,44 @@ open class PurchaseOrderLineService(
         return Pair(count, affectedItemIds)
     }
 
+    /**
+     * Same M18 line id can appear on multiple local PO lines after repeated sync (new m18_data_log each run).
+     * Keep the newest line (highest id) per m18DataLog.m18Id and mark the rest deleted.
+     * @return Pair of (number deleted, itemIds touched for pricing refresh)
+     */
+    open fun markDuplicatePurchaseOrderLinesForSameM18LineId(
+        purchaseOrderId: Long,
+        existingM18LineIds: Set<Long>
+    ): Pair<Int, Set<Long>> {
+        val linesFromM18 =
+            purchaseOrderLineRepository.findAllByPurchaseOrderIdAndDeletedIsFalseAndM18DataLogIsNotNull(purchaseOrderId)
+        val byM18LineId = linesFromM18
+            .mapNotNull { line ->
+                val mid = line.m18DataLog?.m18Id ?: return@mapNotNull null
+                if (mid !in existingM18LineIds) return@mapNotNull null
+                mid to line
+            }
+            .groupBy({ it.first }, { it.second })
+
+        val affectedItemIds = mutableSetOf<Long>()
+        val toSoftDelete = mutableListOf<PurchaseOrderLine>()
+        for ((_, lines) in byM18LineId) {
+            if (lines.size <= 1) continue
+            val sorted = lines.sortedByDescending { it.id ?: 0L }
+            val dupes = sorted.drop(1)
+            dupes.forEach { line ->
+                line.item?.id?.let { affectedItemIds.add(it) }
+                line.deleted = true
+            }
+            toSoftDelete.addAll(dupes)
+        }
+        if (toSoftDelete.isNotEmpty()) {
+            purchaseOrderLineRepository.saveAll(toSoftDelete)
+            purchaseOrderLineRepository.flush()
+        }
+        return Pair(toSoftDelete.size, affectedItemIds)
+    }
+
     open fun findAllPoLineInfoByPoId(poId: Long): List<PurchaseOrderLineInfo> {
         return purchaseOrderLineRepository.findAllPurchaseOrderLineInfoByPurchaseOrderIdAndDeletedIsFalse(poId)
     }