Re: [Quickfix-developers] C# Memory growing fast

SourceForge Headquarters 225 Broadway Suite 1600 San Diego, CA 92101 +1 (858) 422-6466

Ariel,

I've changed the code as follow: (1 month and everthing is going fine)

quickfix\src\C++\MessageSorters.cpp:

/****************************************************************************
** Copyright (c) quickfixengine.org  All rights reserved.
**
** This file is part of the QuickFIX FIX Engine
**
** This file may be distributed under the terms of the quickfixengine.org
** license as defined by quickfixengine.org and appearing in the file
** LICENSE included in the packaging of this file.
**
** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
**
** See http://www.quickfixengine.org/LICENSE for licensing information.
**
** Contact as...@qu... if any conditions of this licensing are
** not clear to you.
**
****************************************************************************/

#ifdef _MSC_VER
#include "stdafx.h"
#else
#include "config.h"
#endif
#include "CallStack.h"

#include "MessageSorters.h"

namespace FIX
{
message_order::message_order( int first, ... )
: m_mode( group ), m_delim( 0 ), m_dictionaryOrder( 0 )
{
  int field = first;
  int size = 0;

  va_list arguments;
  va_start( arguments, first );
  while( field != 0 )
  {
    size++;
    field = va_arg( arguments, int );
  }

  va_start( arguments, first );
  int* order = new int[size];
  order[0] = first;
  for ( int i = 1; i < size; ++i )
    order[ i ] = va_arg( arguments, int );
  setOrder(size, order);
  delete [] order;

  va_end( arguments );
}

message_order::message_order( const int order[] )
: m_mode( group ), m_delim( 0 ), m_dictionaryOrder( 0 )
{
  int size = 0;
  while( order[size] != 0 ) { ++size; }
  setOrder(size, order);
}

message_order& message_order::operator=( const message_order& rhs )
{ QF_STACK_PUSH(message_order::operator=)
  m_mode = rhs.m_mode;
  m_delim = rhs.m_delim;

  if( rhs.m_dictionaryOrder )
  {
	  if( m_dictionaryOrder ) delete [] m_dictionaryOrder;
	  m_dictionaryOrderSize = rhs.m_dictionaryOrderSize;
	  m_dictionaryOrder = new int[ m_dictionaryOrderSize + 1];
	  memcpy( m_dictionaryOrder, rhs.m_dictionaryOrder, ( m_dictionaryOrderSize + 1 ) * sizeof( int ) );
  }

  return *this;

  QF_STACK_POP
}

void message_order::setOrder( int size, const int order[] )
{ QF_STACK_PUSH(message_order::operator=)

  if(size < 1) return;
  m_delim = order[0];
  m_dictionaryOrderSize = size;

  m_dictionaryOrder = new int[ m_dictionaryOrderSize + 1 ];
  memset( m_dictionaryOrder, 0, ( m_dictionaryOrderSize + 1 ) * sizeof( int ) );
  for( int i = 0; i < m_dictionaryOrderSize; i++ )
	  m_dictionaryOrder[ i ] = order[ i ];

  QF_STACK_POP
}
}

quickfix\src\C++\MessageSorters.h

/* -*- C++ -*- */

/****************************************************************************
** Copyright (c) quickfixengine.org  All rights reserved.
**
** This file is part of the QuickFIX FIX Engine
**
** This file may be distributed under the terms of the quickfixengine.org
** license as defined by quickfixengine.org and appearing in the file
** LICENSE included in the packaging of this file.
**
** This file is provided AS IS with NO WARRANTY OF ANY KIND, INCLUDING THE
** WARRANTY OF DESIGN, MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
**
** See http://www.quickfixengine.org/LICENSE for licensing information.
**
** Contact as...@qu... if any conditions of this licensing are
** not clear to you.
**
****************************************************************************/

#ifndef FIX_MESSAGESORTERS_H
#define FIX_MESSAGESORTERS_H

#ifdef _MSC_VER
#pragma warning( disable : 4503 4355 4786 4290 )
#endif

#include "FieldNumbers.h"
#include <stdarg.h>
#include <functional>
#include <map>

namespace FIX
{
/// Sorts fields in correct header order.
struct header_order
{
  static bool compare( const int& x, const int& y )
  {
    int orderedX = getOrderedPosition( x );
    int orderedY = getOrderedPosition( y );

    if ( orderedX && orderedY )
      return orderedX < orderedY;
    else
      if ( orderedX )
        return true;
      else
        if ( orderedY )
          return false;
        else
          return x < y;
  }

  static int getOrderedPosition( const int& field )
  {
    switch ( field )
    {
      case FIELD::BeginString: return 1;
      case FIELD::BodyLength: return 2;
      case FIELD::MsgType: return 3;
      default: return 0;
    };
  }
};

/// Sorts fields in correct trailer order.
struct trailer_order
{
  static bool compare( const int x, const int y )
  {
    if ( x == FIELD::CheckSum ) return false;
    else
      if ( y == FIELD::CheckSum ) return true;
      else return x < y;
  }
};

/// Sorts fields in correct group order
struct group_order
{
  static bool compare( const int x, const int y, int* order, int largest )
  {
    if ( x <= largest && y <= largest )
    {
      int iX = order[ x ];
      int iY = order[ y ];
      if ( iX == 0 && iY == 0 )
        return x < y;
      else if ( iX == 0 )
        return false;
      else if ( iY == 0 )
        return true;
      else
        return iX < iY;
    }
    else if ( x <= largest ) return true;
    else if ( y <= largest ) return false;
    else return x < y;
  }
};

struct dictionaryGroup_order
{
  static bool compare( const int x, const int y, int* order, int orderSize )
  {
	  if(x == y) return false;

	  for(int i = 0; i < orderSize; i++)
	  {
		  if(order[ i ] == x)
			  return true;
		  if(order[ i ] == y)
			  return false;
	  }
	  return x < y;
  }
};

typedef std::less < int > normal_order;

/**
 * Sorts fields in header, normal, or trailer order.
 *
 * Used as a dynamic sorter to create Header, Trailer, and Message
 * FieldMaps while maintaining the same base type.
 */
struct message_order
{
public:
  enum cmp_mode { header, trailer, normal, group };

  message_order( cmp_mode mode = normal ) : m_mode( mode ), m_dictionaryOrder( 0 ) {}
  message_order( int first, ... );
  message_order( const int order[] );
  message_order( const message_order& copy ) : m_dictionaryOrder( 0 )
  { *this = copy; }

  virtual ~message_order()
  {
	  if( m_dictionaryOrder )
		  delete [] m_dictionaryOrder;
  }

  bool operator() ( const int& x, const int& y ) const
  {
    switch ( m_mode )
    {
      case header:
      return header_order::compare( x, y );
      case trailer:
      return trailer_order::compare( x, y );
	  case group:
	  return dictionaryGroup_order::compare(x, y, m_dictionaryOrder, m_dictionaryOrderSize);
      case normal: default:
      return x < y;
    }
  }

  message_order& operator=( const message_order& rhs );

private:
  void setOrder( int size, const int order[] );

  cmp_mode m_mode;
  int m_delim;
  int* m_dictionaryOrder;
  int m_dictionaryOrderSize;
};
}

#endif //FIX_MESSAGESORTERS_H

Fernando Bristotti

________________________________

From: Ariel Piñeiro [mailto:ari...@gm...] 
Sent: Tuesday, March 03, 2009 2:50 PM
To: Bristotti, Fernando
Cc: qui...@li...
Subject: Re: [Quickfix-developers] C# Memory growing fast

have the same problem and I do not know how to fix it!!!!!!!!!!!!!!!

Memory leaks problem.

2009/3/3 Bristotti, Fernando <fer...@cr...>

	- Mostrar texto citado -
	QuickFIX Documentation: http://www.quickfixengine.org/quickfix/doc/html/index.html
	QuickFIX Support: http://www.quickfixengine.org/services.html

	Shane,

	The problem is in the way C++ allocates memory for keeping field order ( struct message_order ):

		quickfix\src\C++\MessageSorters.cpp:

	void message_order::setOrder( int size, const int order[] )
	{ QF_STACK_PUSH(message_order::operator=)

	  if(size < 1) return;
	  m_largest = m_delim = order[0];

	  int* fields = new int[ size + 1 ];
	  fields[ 1 ] = m_delim;
	  // collect all fields and find the largest field number
	  int i;
	  for ( i = 2; i <= size; ++i )
	  {
	    int field = order[i-1];
	    m_largest = m_largest > field ? m_largest : field;
	    fields[ i ] = field;
	  }

	  // populate array with field number as key and position as value
	  m_groupOrder = new int[ m_largest + 1 ];
	  memset( m_groupOrder, 0, ( m_largest + 1 ) * sizeof( int ) );
	  for ( i = 1; i <= size; ++i )
	    m_groupOrder[ fields[ i ] ] = i;
	  delete [] fields;

	  QF_STACK_POP
	}

	Let's suppose that you have a group with only two custom fields ( 10000, 10001 ) in a give fix message. To me, it doesn't make any sence to allocate a int[] vector size of the largest field (10001 in this case) for each entry of this group. Memory will grow really fast!!!!!

	Fernando Bristotti

________________________________

	From: Shane Trotter [mailto:str...@co...] 
	Sent: Tuesday, March 03, 2009 12:15 PM
	To: Bristotti, Fernando
	Cc: qui...@li...
	Subject: Re: [Quickfix-developers] C# Memory growing fast

	I believe with the .NET version you must dispose each message after you have used it.  Is it a requirement in your application to maintain a reference to all 35,000 trade messages?

	--
	Shane Trotter
	Connamara Systems, LLC

	On Tue, Mar 3, 2009 at 9:06 AM, Bristotti, Fernando <fer...@cr...> wrote:

		QuickFIX Documentation: http://www.quickfixengine.org/quickfix/doc/html/index.html
		QuickFIX Support: http://www.quickfixengine.org/services.html

		Hi, 

		I am currently using QuickFIX 1.12.4. Memory grows fast when QuickFIX process a market data snapshot message, for instance the day I've received a snapshot with about 35000 trades (attached), memory grows to about 1.0G and my application crashes.

		Taking a look on QuickFIX source code I could see that struct message_order ( MessageSorters.cpp ) allocates for each group a vector int[] size of the largest field in the group even if the group contains only two fields. The largest field in NoMDEntry group for BM&F market data is a custom field = 6032, thus for processing the message above QuickFIX allocates 6032 x ( 35000 trades ) x ( 4 bytes ).

		I have changed the code in order to allocate a vector int[] size of the number of fields in the group, any suggestion? 

		Thanks, 
		Fernando 

				==============================================================================
		Please access the attached hyperlink for an important electronic communications disclaimer: 

		http://www.credit-suisse.com/legal/en/disclaimer_email_ib.html
		==============================================================================

		------------------------------------------------------------------------------
		Open Source Business Conference (OSBC), March 24-25, 2009, San Francisco, CA
		-OSBC tackles the biggest issue in open source: Open Sourcing the Enterprise
		-Strategies to boost innovation and cut costs with open source participation
		-Receive a $600 discount off the registration fee with the source code: SFAD
		http://p.sf.net/sfu/XcvMzF8H
		_______________________________________________
		Quickfix-developers mailing list
		Qui...@li...
		https://lists.sourceforge.net/lists/listinfo/quickfix-developers

		==============================================================================
	Please access the attached hyperlink for an important electronic communications disclaimer: 

	http://www.credit-suisse.com/legal/en/disclaimer_email_ib.html
	==============================================================================

	------------------------------------------------------------------------------
	Open Source Business Conference (OSBC), March 24-25, 2009, San Francisco, CA
	-OSBC tackles the biggest issue in open source: Open Sourcing the Enterprise
	-Strategies to boost innovation and cut costs with open source participation
	-Receive a $600 discount off the registration fee with the source code: SFAD
	http://p.sf.net/sfu/XcvMzF8H
	_______________________________________________
	Quickfix-developers mailing list
	Qui...@li...
	https://lists.sourceforge.net/lists/listinfo/quickfix-developers

==============================================================================
Please access the attached hyperlink for an important electronic communications disclaimer: 

http://www.credit-suisse.com/legal/en/disclaimer_email_ib.html
==============================================================================