虽然这不是最佳的（因为你必须存储额外的数据/指针，可能会搞砸），你可以把局部变量（或者根据大小和引用的情况）和你的值一起存储：

use std::cmp::{PartialOrd, Ord, Ordering};
use std::collections::BTreeSet;

fn main() {
    dbg!(by_distance_from(-5));
}

fn by_distance_from(x: i32) -> Vec<i32> {
    let v = vec![-5, 0, 1, 3, 10];
    struct Comp<'a> {
        cmp: &'a Cmp,
        v: i32,
    }
    struct Cmp {
        x: i32,
    }
    let cmp = Cmp { x };
    impl PartialOrd for Comp<'_> {
        fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
            self.v.abs_diff(self.cmp.x).partial_cmp(&other.v.abs_diff(other.cmp.x))
        }
    }
    impl Ord for Comp<'_> {
        fn cmp(&self, other: &Self) -> Ordering {
            self.partial_cmp(other).unwrap()
        }
    }
    impl PartialEq for Comp<'_> {
        fn eq(&self, other: &Self) -> bool {
            self.v == other.v
        }
    }
    impl Eq for Comp<'_> {}
    let s: BTreeSet<_> = v.into_iter().map(|v| Comp {
        cmp: &cmp,
        v,
    }).collect();
    s.into_iter().map(|Comp { v, ..}| v).collect()
}

Playground