diff --git a/crates/loro-internal/src/diff_calc.rs b/crates/loro-internal/src/diff_calc.rs
index e933e36d..6407c656 100644
--- a/crates/loro-internal/src/diff_calc.rs
+++ b/crates/loro-internal/src/diff_calc.rs
@@ -2,6 +2,7 @@ use std::{num::NonZeroU16, sync::Arc};
 
 #[cfg(feature = "counter")]
 mod counter;
+use bytes::Bytes;
 #[cfg(feature = "counter")]
 pub(crate) use counter::CounterDiffCalculator;
 pub(super) mod tree;
@@ -1016,7 +1017,11 @@ impl DiffCalculatorTrait for MovableListDiffCalculator {
                             del.is_reversed(),
                         );
                     }
-                    InnerListOp::Move { from, elem_id: from_id, to } => {
+                    InnerListOp::Move {
+                        from,
+                        elem_id: from_id,
+                        to,
+                    } => {
                         // TODO: PERF: this lookup can be optimized
                         let list = oplog
                             .op_groups
@@ -1093,11 +1098,15 @@ impl DiffCalculatorTrait for MovableListDiffCalculator {
                         let mut new_insert = SmallVec::with_capacity(len);
                         for i in 0..len {
                             let id = id.inc(i as i32);
+                            let op = oplog.get_op(id.id()).unwrap();
+                            let elem_id = match op.content.as_list().unwrap() {
+                                InnerListOp::Insert { .. } => id.idlp().compact(),
+                                InnerListOp::Move { elem_id, .. } => elem_id.compact(),
+                                _ => unreachable!(),
+                            };
+
                             // add the related element id
-                            element_changes.insert(
-                                group.get_elem_from_pos(id.idlp()).compact(),
-                                ElementDelta::placeholder(),
-                            );
+                            element_changes.insert(elem_id, ElementDelta::placeholder());
                             new_insert.push(id);
                         }
 
@@ -1114,12 +1123,6 @@ impl DiffCalculatorTrait for MovableListDiffCalculator {
                 .collect(),
         };
 
-        let group = oplog
-            .op_groups
-            .get(&self.container_idx)
-            .unwrap()
-            .as_movable_list()
-            .unwrap();
         element_changes.retain(|id, change| {
             let id = id.to_id();
             // It can be None if the target does not exist before the `to` version
diff --git a/crates/loro-internal/src/history_cache.rs b/crates/loro-internal/src/history_cache.rs
index 354f9b73..5af981d1 100644
--- a/crates/loro-internal/src/history_cache.rs
+++ b/crates/loro-internal/src/history_cache.rs
@@ -133,7 +133,6 @@ impl HistoryCache {
             HistoryCache::MovableList(m) => HistoryCache::MovableList(MovableListOpGroup {
                 arena: a.clone(),
                 elem_mappings: m.elem_mappings.clone(),
-                pos_to_elem: m.pos_to_elem.clone(),
             }),
         }
     }
@@ -279,8 +278,6 @@ pub(crate) struct MovableListOpGroup {
     arena: SharedArena,
     /// mappings from elem_id to a set of target poses & values
     elem_mappings: FxHashMap<IdLp, MovableListTarget>,
-    /// mappings from pos to elem_id
-    pos_to_elem: FxHashMap<IdLp, IdLp>,
 }
 
 #[derive(Debug, Clone)]
@@ -399,7 +396,6 @@ impl OpGroupTrait for MovableListOpGroup {
                         lamport: full_id.lamport,
                         peer: full_id.peer,
                     });
-                    self.pos_to_elem.insert(full_id.idlp(), *from_id);
                 }
                 // Don't mark deletions for now, but the cost is the state now may contain invalid elements
                 // that are deleted but not removed from the state.
@@ -421,7 +417,6 @@ impl MovableListOpGroup {
     fn new(arena: SharedArena) -> Self {
         Self {
             arena,
-            pos_to_elem: Default::default(),
             elem_mappings: Default::default(),
         }
     }
@@ -477,10 +472,6 @@ impl MovableListOpGroup {
                 .cloned(),
         })
     }
-
-    pub(crate) fn get_elem_from_pos(&self, pos: IdLp) -> IdLp {
-        self.pos_to_elem.get(&pos).cloned().unwrap_or(pos)
-    }
 }
 
 #[derive(Default, Clone, Debug)]