How to exclude posts of a certain format from the feed

If you want to modify the feed, you should hook into the main query that WordPress will do on every page request. The best hook here is pre_get_posts. This code example will hook into pre_get_posts, check whether it is a feed, and add the post format taxonomy query:

add_action( 'pre_get_posts', 'wpse18412_pre_get_posts' );
function wpse18412_pre_get_posts( &$wp_query )
{
    if ( $wp_query->is_feed() ) {
        $post_format_tax_query = array(
            'taxonomy' => 'post_format',
            'field' => 'slug',
            'terms' => 'post-format-image', // Change this to the format you want to exclude
            'operator' => 'NOT IN'
        );
        $tax_query = $wp_query->get( 'tax_query' );
        if ( is_array( $tax_query ) ) {
            $tax_query = $tax_query + $post_format_tax_query;
        } else {
            $tax_query = array( $post_format_tax_query );
        }
        $wp_query->set( 'tax_query', $tax_query );
    }
}