Solved it. Just remeber to add the same shipping class to all variations too.
add_filter( 'woocommerce_package_rates', 'custom_shipping_rates', 100, 2 );
function custom_shipping_rates( $rates, $package ) {
$shipping_class = 506; // HERE set the shipping class ID
$found = false;
$productsincart = count($package['contents']);
foreach( $package['contents'] as $cart_item ) {
if ($productsincart == 1 && $cart_item['data'];->get_shipping_class_id() == $shipping_class){
$found = true;
}
}
if ( $found ) {
return false; // If not found we exit
}else {
return $rates;
}
}