I have a working solution now… finally!
<?php
foreach ( $categories as $category ) {
// If there is only one post available, go directly to the post
if($category->count == 1) {
$all_posts = get_posts($category);
echo '<div class="item"><h4 class="item-title">' . get_the_title($all_posts[0]->ID) . '</h4><a href="' . get_permalink($all_posts[0]->ID) . '">Read more</a></div>';
} else {
echo '<div class="item"><h4 class="item-title">' . $category->name . '</h4><a href="' . get_category_link( $category->term_id ) . '">Read more</a></div>';
}
}
?>