Get post title by Alphabet

You have two problems. First need to global $wpdb;. Then you are using the_title() and the_permalink() both of which automatically echo out a value. So you need to switch them to get_the_title() and get_permalink().

function get_post_by_alphabet($the_char){

global $wpdb;

$first_char = $the_char;

$postids=$wpdb->get_col($wpdb->prepare("
SELECT      ID
FROM        $wpdb->posts
WHERE       SUBSTR($wpdb->posts.post_title,1,1) = %s
ORDER BY    $wpdb->posts.post_title",$first_char)); 

if ($postids) {
$args=array(
  'post__in' => $postids,
  'post_status' => 'publish',
  'posts_per_page' => -1,
  'caller_get_posts'=> 1
);
$my_query = null;
$my_query = new WP_Query($args);
if( $my_query->have_posts() ) {
 echo 'List of Posts Titles beginning with the letter '. $first_char;
  while ($my_query->have_posts()) : $my_query->the_post();
    echo '<li><a href="'.get_permalink().'" rel="bookmark" title="Permanent Link to '.the_title_attribute().'"><'.get_the_title().'</a></li>';
  endwhile;
}
wp_reset_query();  // Restore global post data stomped by the_post().
}
}

get_post_by_alphabet('H');