Control the srcset much more (not all sizes in it each time)

There is a filter max_srcset_image_width that lets you limit the sizes used in a srcset attribute to those that are less than a given maximum width.

This code will make it so that only sizes < 500 pixels wide will be used in the srcset:

/**
 * @param int   $max_width  The maximum image width to be included in the 'srcset'. Default '1600'.
 * @param array $size_array Array of width and height values in pixels (in that order).
 */
function wpse_316853_srcset_maximum_width( $max_srcset_image_width, $sizes_array ) {
    return 500;
}
add_filters( 'max_srcset_image_width', 'wpse_316853_srcset_maximum_width', 10, 2 );

In your case though, you might only want to apply this limit when the original image is your 250px wide thumbnail. $sizes_array contains the sizes of the current image, so you can use that to check:

/**
 * @param int   $max_width  The maximum image width to be included in the 'srcset'. Default '1600'.
 * @param array $size_array Array of width and height values in pixels (in that order).
 */
function wpse_316853_srcset_maximum_width( $max_srcset_image_width, $sizes_array ) {
    if ( $sizes_array[0] === 250 ) {
        $max_srcset_image_width = 500;
    }

    return $max_srcset_image_width;
}
add_filters( 'max_srcset_image_width', 'wpse_316853_srcset_maximum_width' );

Or you could do it dynamically, so that the srcset will only include images up to 2x the width of the original image:

/**
 * @param int   $max_width  The maximum image width to be included in the 'srcset'. Default '1600'.
 * @param array $size_array Array of width and height values in pixels (in that order).
 */
function wpse_316853_srcset_maximum_width( $max_srcset_image_width, $sizes_array ) {
    return $sizes_array[0] * 2;
}
add_filters( 'max_srcset_image_width', 'wpse_316853_srcset_maximum_width' );