get term by id without taxonomy

Yes, but you need you make your own SQL query.

A modified version of WP’s get_term():

function &get_term_by_id_only($term, $output = OBJECT, $filter="raw") {
    global $wpdb;
    $null = null;

    if ( empty($term) ) {
        $error = new WP_Error('invalid_term', __('Empty Term'));
        return $error;
    }

    if ( is_object($term) && empty($term->filter) ) {
        wp_cache_add($term->term_id, $term, 'my_custom_queries');
        $_term = $term;
    } else {
        if ( is_object($term) )
            $term = $term->term_id;
        $term = (int) $term;
        if ( ! $_term = wp_cache_get($term, 'my_custom_queries') ) {
            $_term = $wpdb->get_row( $wpdb->prepare( "SELECT t.* FROM $wpdb->terms AS t WHERE t.term_id = %s LIMIT 1", $term) );
            if ( ! $_term )
                return $null;
            wp_cache_add($term, $_term, 'my_custom_queries');
        }
    }

    if ( $output == OBJECT ) {
        return $_term;
    } elseif ( $output == ARRAY_A ) {
        $__term = get_object_vars($_term);
        return $__term;
    } elseif ( $output == ARRAY_N ) {
        $__term = array_values(get_object_vars($_term));
        return $__term;
    } else {
        return $_term;
    }
}

Probably not a good idea. Try to handle the tax too in your meta fields…