You can check if a query has posts with if( $your_query->have_posts() )
, and only output the container if it passes that test.
$featuredPosts = new WP_Query( array(
'posts_per_page' => 2,
'cat' => 6,
'offset' => 4
) );
if ( $featuredPosts->have_posts() ) :
echo('<ul class="right-hang">');
while ( $featuredPosts->have_posts() ) :
$featuredPosts->the_post();
endwhile;
echo '</ul>';
endif;