How can I modify admin bar only if my plugin is loaded

You need to remember screen id of your page and check current screen id before adding new menu item:

add_action( 'admin_menu', 'my_plugin_menu' );
function my_plugin_menu() {
    global $my_swell_plugin_page;
    $my_swell_plugin_page = add_menu_page( 'My Swell Plugin', 'My Swell Plugin', 'edit_posts', 'blah', 'generate_main_page' );
}

function generate_main_page() {
    //We are definitely in our plugins page now - let's modify the admin toolbar.
}

add_action( 'admin_bar_menu', 'add_toolbar_items', 100 );
function add_toolbar_items( $admin_bar ) {
    global $my_swell_plugin_page;

    if ( $my_swell_plugin_page == get_current_screen()->id ) {
        $admin_bar->add_menu( array(
            'id' => 'my-item',
            'title' => 'My Item',
            'href' => '#',
            'meta' => array(
                'title' => __( 'My Item' ),
            ),
        ) );
    }
}