Just create an array with your classes and loop through alongside with the main loop:
function custom_top_products() {
$classes = array( 'first', 'second', 'third', 'fourth', 'fifth' );
$classes_count = count( $classes );
$posts = new WP_Query( array(
'post_type' => 'products',
'post_status' => 'publish',
'orderby' => 'comment_count',
'posts_per_page' => 5
) );
$i = 0;
?><div id="topproducts" class="ratingbox">
<h3 class="rbh3">Top 5 Products</h3>
<div class="topproductcontainer"><?php
if ( $posts->have_posts() ) {
while ( $posts->have_posts() ) {
$posts->the_post();
?><li class="topfive <?php echo $classes[$i++ % $classes_count] ?>">
<a href="https://wordpress.stackexchange.com/questions/152643/<?php the_permalink(); ?>" title="<?php the_title_attribute(); ?>"><?php the_title(); ?></a>
</li><?php
}
wp_reset_postdata();
} else {
echo '<p class="trc">Sorry, No Popular Products Found</p>';
}
?></div>
</div><?php
}