Delivering a file instead of wordpress page

You’re on the right path with a rewrite rule. If you’re just delivering a file, you can hook an early action, check if a download was requested, then output the file and exit before WordPress ever gets to the stage of querying the database and deciding the request is a 404.

// add the download ID query var which we'll set in the rewrite rule
function wpd_query_var( $query_vars ){
    $query_vars[] = 'download_id';
    return $query_vars;
}
add_filter( 'query_vars', 'wpd_query_var' );

// add rewrite rule
function wpd_rewrite(){
    add_rewrite_rule(
        'download/([0-9]+)/?$',
        'index.php?download_id=$matches[1]',
        'top'
    );
}
add_action( 'init', 'wpd_rewrite' );

// check for download_id and output the file
function wpd_request( $wp ){
    if( array_key_exists( 'download_id', $wp->query_vars ) ) {
        // output your headers and file here
        // ID is in $wp->query_vars['download_id']
        // then exit to halt any further execution
        exit;
    }
}
add_action( 'parse_query', 'wpd_request' );