Customise the add media pop-up to include rel attribute option

This is a tough one, but I would start here: Add new “Insert Into Post” button with another function. Below is some code to get started with adding the field via http://rider.sofarider.com/wordpress-tips-and-tricks/extra-input-field-to-add-an-image-panel/ From there, try something like the second code block.

<?php
function attachment_url_extra( $form_fields, $post ) {
    // input field relates to attachments
        // my_field and _my_field is what you should replace with your own
    $post->post_type == 'attachment';
    $form_fields[ 'my_field' ] = array(
        'label' => __( 'MY FIELD' ),
        'input' => 'text',
        'value' => get_post_meta( $post->ID, '_my_field', true )
    );
    $form_fields[ 'my_field' ][ 'label' ] = __( 'MY FIELD' );
    $form_fields[ 'my_field' ][ 'input' ] = 'text';
    $form_fields[ 'my_field' ][ 'value' ] = get_post_meta( $post->ID, '_my_field', true );

    return $form_fields;
}

add_filter( 'attachment_fields_to_edit', 'attachment_url_extra', NULL, 2 );

function attachment_url_extra_save( $post, $attachment ) {

    if( isset( $attachment[ 'my_field' ] ) ) {
        if( trim( $attachment[ 'my_field'] ) == '' ) $post[ 'errors' ][ 'my_field' ][ 'errors' ][] = __( 'Error! Something went wrong.' );
        else update_post_meta( $post[ 'ID' ], '_my_field', $attachment[ 'my_field' ] );
    }
    return $post;

}

Now customizing the actual insert.

add_filter( 'attachment_fields_to_save', 'attachment_url_extra_save', NULL, 2 );
?>

add_filter('media_send_to_editor', 'my_filter_iste', 20, 3);

function my_filter_iste($html, $id, $caption, $title, $align, $url, $size, $alt) {
    $attachment = get_post($id); //fetching attachment by $id passed through

    $mime_type = $attachment->post_mime_type; //getting the mime-type
    if (substr($mime_type, 0, 5) == 'video') { //checking mime-type
        $src = wp_get_attachment_url( $id );
        $html="';  
    }
    return $html; // return new $html
}