Fix stack overflow and other memory corruption problems with FC6 on Dell 2950

git-svn-id: http://svn.asterisk.org/svn/octasic_api/oct612x/trunk@25 537310ab-6354-42db-a3cc-247b777f7be6

2 files changed, 33 insertions, 14 deletions

diff --git a/software/octdeviceapi/oct6100api/oct6100_api/oct6100_channel.c b/software/octdeviceapi/oct6100api/oct6100_api/oct6100_channel.c
index 43bb1ed..c0c9d2a 100644
--- a/software/octdeviceapi/oct6100api/oct6100_api/oct6100_channel.c
+++ b/software/octdeviceapi/oct6100api/oct6100_api/oct6100_channel.c
@@ -1293,30 +1293,40 @@ UINT32 Oct6100ChannelOpenSer(
 				IN tPOCT6100_INSTANCE_API			f_pApiInstance,
 				IN OUT tPOCT6100_CHANNEL_OPEN		f_pChannelOpen )
 {
-	tOCT6100_API_ECHO_CHAN_INDEX		ChannelIndexConf;
+	tOCT6100_API_ECHO_CHAN_INDEX		*ChannelIndexConf;
 	UINT32	ulResult;
 
+	ChannelIndexConf = kmalloc(sizeof(*ChannelIndexConf), GFP_KERNEL);
+
+	if (!ChannelIndexConf)
+		return cOCT6100_ERR_FATAL_0;
+
 	/* Check the user's configuration of the echo cancellation channel for errors. */
-	ulResult = Oct6100ApiCheckChannelParams( f_pApiInstance, f_pChannelOpen, &ChannelIndexConf );
+	ulResult = Oct6100ApiCheckChannelParams( f_pApiInstance, f_pChannelOpen, ChannelIndexConf );
 	if ( ulResult != cOCT6100_ERR_OK  )
-		return ulResult;
+		goto out;
 
 	/* Reserve all resources needed by the echo cancellation channel. */
-	ulResult = Oct6100ApiReserveChannelResources( f_pApiInstance, f_pChannelOpen, &ChannelIndexConf );
+	ulResult = Oct6100ApiReserveChannelResources( f_pApiInstance, f_pChannelOpen, ChannelIndexConf );
 	if ( ulResult != cOCT6100_ERR_OK  )
-		return ulResult;
+		goto out;
 
 	/* Write all necessary structures to activate the echo cancellation channel. */
-	ulResult = Oct6100ApiWriteChannelStructs( f_pApiInstance, f_pChannelOpen, &ChannelIndexConf );
+	ulResult = Oct6100ApiWriteChannelStructs( f_pApiInstance, f_pChannelOpen, ChannelIndexConf );
 	if ( ulResult != cOCT6100_ERR_OK  )
-		return ulResult;
+		goto out;
 
 	/* Update the new echo cancellation channels's entry in the ECHO channel list. */
-	ulResult = Oct6100ApiUpdateChannelEntry( f_pApiInstance, f_pChannelOpen, &ChannelIndexConf );
+	ulResult = Oct6100ApiUpdateChannelEntry( f_pApiInstance, f_pChannelOpen, ChannelIndexConf );
 	if ( ulResult != cOCT6100_ERR_OK  )
-		return ulResult;
+		goto out;
 
+	kfree(ChannelIndexConf);
 	return cOCT6100_ERR_OK;
+
+out:
+	kfree(ChannelIndexConf);
+	return ulResult;
 }
 #endif
 
@@ -3564,7 +3574,7 @@ UINT32 Oct6100ChannelModifySer(
 	/* We don't want this 290 byte structure on the stack */
 	pTempChanOpen = kmalloc(sizeof(*pTempChanOpen), GFP_ATOMIC);
 	if (!pTempChanOpen)
-		return cOCT6100_ERR_CHANNEL_NOT_OPEN;
+		return cOCT6100_ERR_FATAL_0;
 
 	/* Check the user's configuration of the echo cancellation channel for errors. */
 	ulResult = Oct6100ApiCheckChannelModify( f_pApiInstance, 
diff --git a/software/octdeviceapi/oct6100api/oct6100_api/oct6100_chip_open.c b/software/octdeviceapi/oct6100api/oct6100_api/oct6100_chip_open.c
index 3f4baed..36a07c4 100644
--- a/software/octdeviceapi/oct6100api/oct6100_api/oct6100_chip_open.c
+++ b/software/octdeviceapi/oct6100api/oct6100_api/oct6100_chip_open.c
@@ -33,6 +33,8 @@ $Octasic_Revision: 336 $
 
 /*****************************  INCLUDE FILES  *******************************/
 
+#include <linux/slab.h>
+
 #include "octdef.h"
 
 #include "oct6100api/oct6100_defines.h"
@@ -272,7 +274,7 @@ UINT32 Oct6100ChipOpen(
 				tPOCT6100_INSTANCE_API	f_pApiInstance,
 				tPOCT6100_CHIP_OPEN		f_pChipOpen )
 {
-	tOCT6100_API_INSTANCE_SIZES	InstanceSizes;
+	tOCT6100_API_INSTANCE_SIZES	*InstanceSizes;
 	UINT32						ulStructSize;
 	UINT32						ulResult;
 	UINT32						ulTempVar;
@@ -310,13 +312,20 @@ UINT32 Oct6100ChipOpen(
 	if ( ulResult != cOCT6100_ERR_OK )
 		return ulResult;
 
+	InstanceSizes = kmalloc(sizeof(tOCT6100_API_INSTANCE_SIZES), GFP_KERNEL);
+	if (!InstanceSizes)
+		return cOCT6100_ERR_FATAL_0;
 	/* Calculate the amount of memory needed for the API instance structure. */
-	ulResult = Oct6100ApiCalculateInstanceSizes( f_pChipOpen, &InstanceSizes );
-	if ( ulResult != cOCT6100_ERR_OK )
+	ulResult = Oct6100ApiCalculateInstanceSizes( f_pChipOpen, InstanceSizes );
+	if ( ulResult != cOCT6100_ERR_OK ) {
+		kfree(InstanceSizes);
 		return ulResult;
+	}
 
 	/* Allocate the memory for the API instance structure internal pointers. */
-	ulResult = Oct6100ApiAllocateInstanceMemory( f_pApiInstance, &InstanceSizes );
+	ulResult = Oct6100ApiAllocateInstanceMemory( f_pApiInstance, InstanceSizes );
+	kfree(InstanceSizes);
+
 	if ( ulResult != cOCT6100_ERR_OK )
 		return ulResult;