How to delete old custom-css-stuff in the customizer-live-preview?

<style> tag accepts all the global HTML attribute, including class and id. So, you could do something like this:

wp.customize( 'custom_css', function( value ) {
    value.bind( function( newval ) {

      if ( $("#my-custom-style").length > 0 ) {
        $("#my-custom-style").remove();
      }

      $('head').append("<style id="my-custom-style">"+ newval + "</style>");

    } );
} );