you can use a walker like this,
class Walker_Recent_Post_Category extends Walker_Category {
function start_el(&$output, $category, $depth, $args) {
extract($args);
$cat_name = esc_attr( $category->name);
$cat_name = apply_filters( 'list_cats', $cat_name, $category );
$list_recent_cat_post="<ul class="show-hide">";
$args = array( 'numberposts' => 5, 'category_name' => $category->name );
$myposts = get_posts( $args );
foreach( $myposts as $mypost ) :
$list_recent_cat_post .= '<li><a href="' . get_permalink($mypost->ID) . '">' . $mypost->post_title . '</a></li>';
endforeach;
$list_recent_cat_post .= '</ul>';
$link = '<a href="' . get_category_link( $category->term_id ) . '" ';
if ( $use_desc_for_title == 0 || empty($category->description) )
$link .= 'title="' . sprintf(__( 'View all posts filed under %s' ), $cat_name) . '"';
else
$link .= 'title="' . esc_attr( strip_tags( apply_filters( 'category_description', $category->description, $category ) ) ) . '"';
$link .= '>';
$link .= $cat_name . '</a>';
if ( (! empty($feed_image)) || (! empty($feed)) ) {
$link .= ' ';
if ( empty($feed_image) )
$link .= '(';
$link .= '<a href="' . get_category_feed_link($category->term_id, $feed_type) . '"';
if ( empty($feed) )
$alt=" alt="" . sprintf(__( 'Feed for all posts filed under %s' ), $cat_name ) . '"';
else {
$title=" title="" . $feed . '"';
$alt=" alt="" . $feed . '"';
$name = $feed;
$link .= $title;
}
$link .= '>';
if ( empty($feed_image) )
$link .= $name;
else
$link .= "<img src="$feed_image"$alt$title" . ' />';
$link .= '</a>';
if ( empty($feed_image) )
$link .= ')';
}
if ( isset($show_count) && $show_count )
$link .= ' (' . intval($category->count) . ')';
if ( isset($show_date) && $show_date ) {
$link .= ' ' . gmdate('Y-m-d', $category->last_update_timestamp);
}
$link .= $list_recent_cat_post;
if ( isset($current_category) && $current_category )
$_current_category = get_category( $current_category );
if ( 'list' == $args['style'] ) {
$output .= "\t<li";
$class="cat-item cat-item-".$category->term_id;
if ( isset($current_category) && $current_category && ($category->term_id == $current_category) )
$class .= ' current-cat';
elseif ( isset($_current_category) && $_current_category && ($category->term_id == $_current_category->parent) )
$class .= ' current-cat-parent';
$output .= ' class="'.$class.'"';
$output .= ">$link\n";
} else {
$output .= "\t$link\n";
}
}
}
then call the walker like this,
<ul>
<?php wp_list_categories(
array(
'show_count'=>1,
'title_li' =>'',
'walker' => new Walker_Recent_Post_Category()
)
); ?>
</ul>
it will display something like this,
<ul>
<li>
<a href="http://localhost/wordpress/category/my-category/" title="View all posts filed under my category">my category</a> (1)
<ul class="show-hide">
<li><a href="http://localhost/wordpress/2011/06/twetwetewrwe/">twetwetewrwe</a></li>
</ul>
</li>
<li>
<a href="http://localhost/wordpress/category/uncategorized/" title="View all posts filed under Uncategorized">Uncategorized</a> (7)
<ul class="show-hide">
<li><a href="http://localhost/wordpress/2011/06/5th/">5th</a></li>
<li><a href="http://localhost/wordpress/2011/06/twtwe/">twtwe</a></li>
<li><a href="http://localhost/wordpress/2010/08/five-ways-widget-manufacturing-can-be-bungled/">Lorem Ipsum dolor sin amet</a></li>
<li><a href="http://localhost/wordpress/2010/08/help-me/">help-me!</a></li>
<li><a href="http://localhost/wordpress/2010/08/hello-world-2/">Hello world!</a></li>
</ul>
</li>
</ul>
and some jQuery touches on the html : demo