create shortcode to show children if any otherwise siblings

Yes, you’re close. What you want is something like this:

function rt_list_children() {
    global $post;
    $output="";

    if ( $post->post_parent ) {  // if it's a child page
        $siblings = wp_list_pages( array(
            'child_of' => $post->post_parent,
            'title_li'    => '',
            'echo' => '0',
        ) );
        if ( $siblings ) {  // it has siblings
            $children = $siblings;
        } else {  // it has no siblings, so show its children
            $children = wp_list_pages( array(
                'child_of' => $post->ID,
                'title_li'    => '',
                'echo' => '0',
            ) );
        }
    } else {  // it's a top level page, show its children
        $children = wp_list_pages( array(
            'child_of' => $post->ID,
            'title_li'    => '',
            'echo' => '0',
        ) );
    }

    if ( $children ) {
       $output="<ul>" . $children . '</ul>';
    }
    return $output;
}
add_shortcode ('sidebar-menu','rt_list_children');