Thanks to Sally for the answer, which is to use the NOT EXISTS operator:
$args = [
'post_type' => 'post',
'tax_query' => [
'relation' => 'AND',
[
'taxonomy' => 'areaoflondon',
'field' => 'name',
'terms' => 'South London',
'include_children' => false,
],
[
'taxonomy' => 'yearofvisit',
'operator' => 'NOT EXISTS'
]
],
'posts_per_page' => 5,
'meta_key' => 'rating',
'meta_type' => 'NUMERIC',
'order' => 'DESC',
'orderby' => 'meta_value_num',
];