Remove taxonomies using register_post_type_args

Did you try this? Here: unregister_taxonomy_for_object_type()

Read more: https://developer.wordpress.org/reference/functions/unregister_taxonomy_for_object_type/

function unregister_portfolio_cat_for_portfolio() {
    unregister_taxonomy_for_object_type( 'portfolio-cat', 'portfolio' );
}
add_action( 'init', 'unregister_portfolio_cat_for_portfolio' );