Have you ever tried the functions described here? It seems to be exactly what you’re looking after.
Here’s a slightly variation of the function for reference:
// Run only when editing a page
// For new pages load-page-new.php should be used
// See: http://core.trac.wordpress.org/browser/tags/3.5.1/wp-admin/admin.php#L217
add_action( 'load-page.php', 'hide_tinyeditor_wp' );
function hide_tinyeditor_wp() {
// Not really necessary, but just in case
if( !isset( $_GET['post'] ) )
return;
$template = get_post_meta( $_GET['post'] , '_wp_page_template', true );
if($template_file == 'specific-template.php'){ // edit the template name
remove_post_type_support('page', 'editor');
}
}