You have to use the_content
and the_excerpt_rss
hooks. You can do this like this:
function rss_append_footer($content) {
if(is_feed()) {
$related_posts_html = ... // I assume you already can compute it
$content .= $related_posts_html;
}
return $content;
}
add_filter('the_content', 'rss_append_footer');
add_filter('the_excerpt_rss', 'rss_append_footer');