Show the section only if custom taxonomy was chosen

You already check, if given post has any terms assigned. All you need to do is to move that check earlier in the code.

<?php 
    $terms = get_the_terms( get_the_ID(), 'colour' );
    if ( $terms && ! is_wp_error( $terms ) ) :
?>
<section class="colour__chart">
    <div class="container">
        <div id="color-chart">
            <h2>Colour Chart</h2>
            <hr class="divider">
            <?php
                foreach ( $terms as $term ) {
                    echo get_field( 'name', 'colour_' . $term->term_id ); 

                    echo '<div class="col-md-2 col-sm-4 col-xs-4 text-center">';
                    echo '<div class="color-chart__item" style="background-color:' . get_field( 'colour_acf', 'colour_' . $term->term_id ) . '"></div>';
                    echo '<p>' . $term->name . '</p>' ;
                    echo '</div>' ;
                }
            ?>
        </div>
    </div>
</section>
<?php endif; ?>