How to disable users from editing billing address in WooCommerce checkout?

Okay I found a solution after Slack conversation. It is possible to remove the fields from checkout by unsetting them in woocommerce_billing_fields hook and then re-adding the values again to the order from users profile with woocommerce_checkout_posted_data hook.

add_filter( 'woocommerce_checkout_posted_data', 'fill_order_billing_details' );

function fill_order_billing_details( $data ) {
  $customer_id = get_current_user_id();

  $data['billing_first_name'] = get_user_meta( $customer_id, 'billing_first_name', true );
  $data['billing_last_name']  = get_user_meta( $customer_id, 'billing_last_name', true );
  $data['billing_company']    = get_user_meta( $customer_id, 'billing_company', true );
  $data['billing_country']    = get_user_meta( $customer_id, 'billing_country', true );
  $data['billing_address_1']  = get_user_meta( $customer_id, 'billing_address_1', true );
  $data['billing_address_2']  = get_user_meta( $customer_id, 'billing_address_2', true );
  $data['billing_city']       = get_user_meta( $customer_id, 'billing_city', true );
  $data['billing_state']      = get_user_meta( $customer_id, 'billing_state', true );
  $data['billing_postcode']   = get_user_meta( $customer_id, 'billing_postcode', true );
  $data['billing_phone']      = get_user_meta( $customer_id, 'billing_phone', true );
  $data['billing_email']      = get_user_meta( $customer_id, 'billing_email', true );

  return $data;
}