You can hook a filter to the post_thumbnail_html
and check if its a post in that category and if the user is not logged in ex:
add_filter( 'post_thumbnail_html', 'my_post_image_html_wpa92119', 10, 3 );
function my_post_image_html_wpa92119( $html, $post_id, $post_image_id ) {
$category_to_exclude = "CHANGE_WITH_CATEGORY ID";
$logIn_Img_path = "CHANGE THIS WITH PATH/TO/NON-LOGGED/USERS/IMAGE";
//check if the post have that category nad if the user is not logged in
if (has_category( $category_to_exclude ) && !is_user_logged_in())
return '<a href="'.wp_login_url( get_permalink($post_id) ).'"><img src="'.$logIn_Img_path.'" alt="please log in"></a>';
//if you got here then he is either logged in or its not the specified category
return $html;
}