Add Class according to the order of appearance

Just create an array with your classes and loop through alongside with the main loop:

function custom_top_products() {
    $classes = array( 'first', 'second', 'third', 'fourth', 'fifth' );
    $classes_count = count( $classes );

    $posts = new WP_Query( array(
        'post_type' => 'products',
        'post_status' => 'publish',
        'orderby' => 'comment_count',
        'posts_per_page' => 5
    ) );

    $i = 0;

    ?><div id="topproducts" class="ratingbox">
        <h3 class="rbh3">Top 5 Products</h3>
        <div class="topproductcontainer"><?php
            if ( $posts->have_posts() ) {
                while ( $posts->have_posts() ) {
                    $posts->the_post();
                    ?><li class="topfive <?php echo $classes[$i++ % $classes_count] ?>">
                        <a href="https://wordpress.stackexchange.com/questions/152643/<?php the_permalink(); ?>" title="<?php the_title_attribute(); ?>"><?php the_title(); ?></a>
                    </li><?php
                }

                wp_reset_postdata();
            } else {
                echo '<p class="trc">Sorry, No Popular Products Found</p>';
            }
            ?></div>
    </div><?php
}