Second transition_post_status hook fired instead of the first

You can user if..else condition to combine both function into one hook.

add_action( 'transition_post_status', 'groupqueue_to_groupcpt_and_groupcpt_to_bannedcpt', 10, 3);
function groupqueue_to_groupcpt_and_groupcpt_to_bannedcpt($new_status, $old_status, $post){
    //Move to Group Post Type
    if ('publish' === $new_status  && 'groupq' === $post->post_type )
    {    
        wp_update_post(array(
            'ID'        => $post->ID,
            'post_type' => 'group'
        ), true );
    }
    else if ( 'banned' === $new_status && ('groupq' === $post->post_type || 'group' === $post->post_type))
    {
        wp_update_post(array(
            'ID'        => $post->ID,
            'post_type' => 'banned'
        ), true );
    }
}