Use the get_the_terms
function instead so you can build the list manually and access the term_id
for each term.
$terms = get_the_terms( $post->ID, 'taxanomy-name' );
echo '<ul>';
foreach( $terms as $term ):
echo '<li>';
echo $term->name;
the_field( 'custom-field-name', 'taxanomy-name_' . $term->term_id );
echo '</li>';
endforeach;
echo '</ul>';