Custom Post Type Navigation on Custom Field

For previous and next post links to work, you need to hook on to the get_*_post_* filters and modify the query to actually sort the posts using your custom field. I hope the code below will work for you.

For Previous post link

add_filter('get_previous_post_join', 'wpse96670_get_previous_post_join_custom_sort');
add_filter('get_previous_post_where', 'wpse96670_get_previous_post_where_custom_sort');
add_filter('get_previous_post_sort', 'wpse96670_get_previous_post_sort_custom_sort');
function wpse96670_get_previous_post_join_custom_sort($join) {
    global $wpdb;
    return "INNER JOIN {$wpdb->prefix}postmeta ON (p.ID = {$wpdb->prefix}postmeta.post_id)";
}
function wpse96670_get_previous_post_where_custom_sort($where) {
    global $wpdb, $post;
    $meta_value = get_post_meta($post->ID, 'aaaConsultant_dateofletter', true);
    return "WHERE 1=1  AND p.post_type="letters_of_ref" AND p.post_status="publish" AND {$wpdb->prefix}postmeta.meta_key = 'aaaConsultant_dateofletter' AND {$wpdb->prefix}postmeta.meta_value < '{$meta_value}'";
}
function wpse96670_get_previous_post_sort_custom_sort($sort) {
    global $wpdb;
    return "ORDER BY {$wpdb->prefix}postmeta.meta_value DESC LIMIT 1";
}

For next post link

add_filter('get_next_post_join', 'wpse96670_get_next_post_join_custom_sort');
add_filter('get_next_post_where', 'wpse96670_get_next_post_where_custom_sort');
add_filter('get_next_post_sort', 'wpse96670_get_next_post_sort_custom_sort');
function wpse96670_get_next_post_join_custom_sort($join) {
    global $wpdb;
    return "INNER JOIN {$wpdb->prefix}postmeta ON (p.ID = {$wpdb->prefix}postmeta.post_id)";
}
function wpse96670_get_next_post_where_custom_sort($where) {
    global $wpdb, $post;
    $meta_value = get_post_meta($post->ID, 'aaaConsultant_dateofletter', true);
    return "WHERE 1=1  AND p.post_type="letters_of_ref" AND p.post_status="publish" AND {$wpdb->prefix}postmeta.meta_key = 'aaaConsultant_dateofletter' AND {$wpdb->prefix}postmeta.meta_value > '{$meta_value}'";
}
function wpse96670_get_next_post_sort_custom_sort($sort) {
    global $wpdb;
    return "ORDER BY {$wpdb->prefix}postmeta.meta_value ASC LIMIT 1";
}

Then you need some code in the template:

    <?php 
        $oNext = get_adjacent_post(false, '', true);
        $oPrev = get_adjacent_post(false, '', false);

        if ($oPrev || $oNext) {
            echo '<div id="lor_nav">' . "\n";
        }
        if ($oPrev) {
            echo '<div class="prevlink"><a href="' . get_permalink($oPrev->ID) . '"> &laquo; ' . $oPrev->post_title . '</a></div>' . "\n";
        }
        if ($oNext) {
            echo '<div class="nextlink"><a href="' . get_permalink($oNext->ID) . '">' . $oNext->post_title . ' &raquo;</a></div>' . "\n";
        }
        if ($oPrev || $oNext) {
            echo '</div><div class="clear"></div>' . "\n";
        }
?>

Let me know if you need any clarification.