I think I figured it out. This query appears to work:
$args = array();
if ( isset( $term_ids ) && isset( $category_ids ) ) {
$args['tax_query'] = array(
'relation' => 'AND',
array(
'taxonomy' => 'category',
'field' => 'term_id',
'terms' => $omit_category_ids,
'operator' => 'NOT IN',
),
array(
'relation' => 'OR',
array(
'taxonomy' => 'category',
'field' => 'term_id',
'terms' => $category_ids,
'operator' => 'IN',
),
array(
'taxonomy' => 'post_tag',
'field' => 'term_id',
'terms' => $term_ids,
'operator' => 'IN',
),
),
);
} elseif ( isset( $term_ids ) ) {
$args['tax_query'] = array(
'relation' => 'AND',
array(
'taxonomy' => 'category',
'field' => 'term_id',
'terms' => $omit_category_ids,
'operator' => 'NOT IN',
),
array(
'taxonomy' => 'post_tag',
'field' => 'term_id',
'terms' => $term_ids,
'operator' => 'IN',
),
);
} elseif ( isset( $category_ids ) ) {
$args['tax_query'] = array(
'relation' => 'AND',
array(
'taxonomy' => 'category',
'field' => 'term_id',
'terms' => $omit_category_ids,
'operator' => 'NOT IN',
),
'relation' => 'OR',
array(
'taxonomy' => 'category',
'field' => 'term_id',
'terms' => $category_ids,
'operator' => 'IN',
),
);
}