Get post-meta value of all custom-posts – lowest to highest year-count?

something like this should do the job:

global $post;
$years = array();
$loop = new WP_Query( $args );
while ( $loop->have_posts() ) : $loop->the_post();
    $this_year = get_post_meta( $post->ID, 'event_date', true );
    if ( $this_year = date( 'Y', $this_year ) ) {
        if ( ! in_array( $this_year, $years ) ) {
            $years[] = $this_year;
        }
    }
endwhile;
rsort( $years ); // sorts the years array into reverse order
echo implode( ' | ', $years );

to reduce queries, you can save everything into a transient:

global $post;
$transient="all-post-years";
$timeout   = 14400; // 4 hours

if ( false === $out = get_transient( $transient ) ) {

    $years = array();
    $loop  = new WP_Query( $args );

    while ( $loop->have_posts() ) : $loop->the_post();
        $this_year = get_post_meta( $post->ID, 'event_date', true );
        if ( $this_year = date( 'Y', $this_year ) ) {
            if ( ! in_array( $this_year, $years ) ) {
                $years[] = $this_year;
            }
        }
    endwhile;

    rsort( $years ); // sorts the years array into reverse order

    foreach ( $years as $year ) {
        $out .= '<a href="#' . $year . '" name="y' . $year . '" class="inactive">' . $year . '</a> <span class="sep">|</span> ';
    }

    if ( $out ) {
        set_transient( $transient, $out, $timeout );
    }

}

echo $out;