Determine which theme location a wp_get_nav_menu_items is for

Not sure this is the simplest means, but you could conditionally add the filter to just the location you want to modify via wp_nav_menu_args filter when wp_nav_menu is called, then immediately remove the filter so it isn’t applied to other menus.

function wpa108544_nav_menu_args( $args ){
    // check if it's the location we want the filter applied to
    if( 'primary' == $args['theme_location'] )
        add_filter( 'wp_get_nav_menu_items', 'wpa108544_get_nav_menu_items', 10, 3 );

    return $args;
}
add_filter( 'wp_nav_menu_args', 'wpa108544_nav_menu_args' );


function wpa108544_get_nav_menu_items( $items, $menu, $args ){
    // remove the filter
    remove_filter( current_filter(), __FUNCTION__, 10, 3 );

    // do your menu manipulation stuff here
}