CXTPDockingPaneManager::m_bCloseGroupOnButtonClick = FALSE;
CXTPDockingPaneManager::m_bHideGroupOnButtonClick = TRUE;

you can patch void CXTPDockingPaneTabbedContainer::OnCaptionButtonClick(CXTPDoc kingPaneCaptionButton* pButton):

if (pButton->GetID() == XTP_IDS_DOCKINGPANE_CLOSE)
 {
  if (pManager->m_bCloseGroupOnButtonClick)
  {
   POSITION pos = m_lstPanes.GetTailPosition();
   while (pos)
   {
    CXTPDockingPane* pPane = (CXTPDockingPane*)m_lstPanes.GetPrev(pos);
    
    if ((pPane->GetOptions() & xtpPaneNoCloseable) != 0)
     continue;
    
    if (!pManager->_OnAction(xtpPaneActionClosing, pPane))
    {    
     pPane->Close();     
     pManager->_OnAction(xtpPane ActionClosed, pPane);
    }
   }

  }
  else
  {  
   if (!pManager->_OnAction(xtpPaneActionClosing, pPane))
   {    
    pPane->Close();     
    pManager->_OnAction(xtpPaneAction Closed, pPane);
   }
  }
 }