A bit involved, but this should work:
function remove_wysiwyg() {
global $pagenow;
if ( 'post.php' == $pagenow ) {
$type = get_post_type( $_GET['post'] );
if( 'post' != $type || 'page' != $type )
add_filter('user_can_richedit', '__return_false');
} elseif ( 'post-new.php' == $pagenow ) {
if( isset( $_GET['post_type'] ) && 'page' != $_GET['post_type'] )
add_filter('user_can_richedit', '__return_false');
}
}
add_action( 'admin_init', 'remove_wysiwyg' );