OK got it. For anyone who needs to do this , it is best you extend Walker_Nav_Menu class if you just need to add things to existing wp menu structure. Otherwise if you are extending Walker class you kinda have to rebuild the menu completely.
Include this class
class Walker_Extend_Menu extends Walker_Nav_Menu {
// Tell Walker where to inherit it's parent and id values
var $db_fields = array(
'parent' => 'menu_item_parent',
'id' => 'db_id'
);
/**
* Starts the list before the elements are added.
*
* @see Walker::start_lvl()
*
* @since 3.0.0
*
* @param string $output Passed by reference. Used to append additional content.
* @param int $depth Depth of menu item. Used for padding.
* @param array $args An array of arguments. @see wp_nav_menu()
*/
public function start_lvl( &$output, $depth = 0, $args = array() ) {
$indent = str_repeat("\t", $depth);
$output .= "\n$indent<div class=\"holder\">\n$indent<ul class=\"sub-menu\">\n";
}
/**
* Ends the list of after the elements are added.
*
* @see Walker::end_lvl()
*
* @since 3.0.0
*
* @param string $output Passed by reference. Used to append additional content.
* @param int $depth Depth of menu item. Used for padding.
* @param array $args An array of arguments. @see wp_nav_menu()
*/
public function end_lvl( &$output, $depth = 0, $args = array() ) {
$indent = str_repeat("\t", $depth);
$output .= "$indent</ul>\n</div>\n";
}
}
And call your walker in wp_nav_menu
$defaults = array(
'theme_location' => 'primary',
'menu' => '',
'container' => 'div',
'container_class' => '',
'container_id' => 'mega-menu',
'menu_class' => 'mega',
'menu_id' => '',
'echo' => true,
'fallback_cb' => 'wp_page_menu',
'before' => '',
'after' => '',
'link_before' => '',
'link_after' => '',
'items_wrap' => '<ul id="%1$s" class="%2$s">%3$s</ul>',
'depth' => 0,
'walker' => new Walker_Extend_Menu ()
);