Well, that will change when I release Shinobi X (if/when I decide to do so). Still doesn't make much of a difference since the FM handling didn't really get worse than what was there before, it just does it wrong in a different way.

I've examined FM a bit more while working with Shinobi X, and while I wasn't able to make any significant progress on getting it to sound correct, I was able to get MDXSL and MDYSL working properly. I've verified that these are correct by comparing their values with the fm_layer and fm_gen values given by ssfinfo and they are consistent with the tables in the SCSP manual. To achieve this, I changed the FM ring buffer pointer to decrement rather than increment as well as making it update regardless of whether a slot is active or not.

While investigating FM issues, I uncovered a few more (relatively minor) problems. I fixed a small LPSLNK problem (wasn't handling this quite correctly before). Also, I made a change to the DSP input mix level, which improves the effect balance. Previously, DSP output was a bit quiet compared to the direct output. I resolved this by shifting left what was the previous input by 2. I believe this is correct since the LPANTABLE/RPANTABLE calculations contain a multiplier of 4.0, and an additional left shift of 2 is in line with the fact that the MIXS input is 20 bits.

Below is a upgraded patch to AOSDK 1.1.6. I've commented out the FM-synth stuff because it isn't correct yet and right now Shinobi X sounds better with it disabled.

Code
diff -Nru aosdk_base/eng_ssf/scsp.c aosdk/eng_ssf/scsp.c
--- aosdk_base/eng_ssf/scsp.c	2007-12-14 09:09:20.000000000 -0800
+++ aosdk/eng_ssf/scsp.c	2007-12-17 07:09:39.000000000 -0800
@@ -73,7 +73,7 @@
 #define SDIR(slot)		((slot->udata.data[0x6]>>0x0)&0x0100)
 #define TL(slot)		((slot->udata.data[0x6]>>0x0)&0x00FF)
 
-#define MDL(slot)		((slot->udata.data[0x7]>>0xB)&0x0007)
+#define MDL(slot)		((slot->udata.data[0x7]>>0xC)&0x000F)
 #define MDXSL(slot)		((slot->udata.data[0x7]>>0x6)&0x003F)
 #define MDYSL(slot)		((slot->udata.data[0x7]>>0x0)&0x003F)
 
@@ -384,7 +384,7 @@
 static UINT32 SCSP_Step(struct _SLOT *slot)
 {
 	int octave=OCT(slot);
-	UINT32 Fn;
+	UINT64 Fn;
 
 	Fn=(FNS_Table[FNS(slot)]);	//24.8
 	if(octave&8)
@@ -1114,11 +1114,16 @@
 		addr2=(slot->nxt_addr>>(SHIFT-1))&0x7fffe;
 	}
 
-	if(MDL(slot)!=0 || MDXSL(slot)!=0 || MDYSL(slot)!=0)
+	/*if(MDL(slot)!=0 || MDXSL(slot)!=0 || MDYSL(slot)!=0)
 	{
 		INT32 smp=(SCSP->RINGBUF[(SCSP->BUFPTR+MDXSL(slot))&63]+SCSP->RINGBUF[(SCSP->BUFPTR+MDYSL(slot))&63])/2;
+		INT32 cycle=LEA(slot)-LSA(slot); // cycle corresponds to 2 pi
 
-		smp>>=11;
+		smp*=cycle; // associate cycle with full 16-bit sample range
+		smp>>=0x1A-MDL(slot); // ex. for MDL=0xF, sample range corresponds to +/- 64 pi (32=2^5 cycles) so shift by 11 (16-5 == 0x1A-0xF)
+		while(smp<0) smp+=cycle; smp%=cycle; // keep modulation sampler within a single cycle
+		if(!PCM8B(slot)) smp<<=1;
+		
 		addr1+=smp; addr2+=smp;
 		if(!PCM8B(slot))
 		{
@@ -1128,13 +1133,7 @@
 		{
 			addr1&=0x7ffff; addr2&=0x7ffff;
 		}
-	}
-
-	if(addr1==LSA(slot))
-	{
-		if(LPSLNK(slot) && slot->EG.state==ATTACK)
-			slot->EG.state = DECAY1;
-	}
+	}*/
 
 	if(PCM8B(slot))	//8 bit signed
 	{
@@ -1171,6 +1170,12 @@
 	addr1=slot->cur_addr>>SHIFT;
 	addr2=slot->nxt_addr>>SHIFT;
 	
+	if(addr1>=LSA(slot) && !(slot->Backwards))
+	{
+		if(LPSLNK(slot) && slot->EG.state==ATTACK)
+			slot->EG.state = DECAY1;
+	}
+	
 	for (addr_select=0;addr_select<2;addr_select++)
 	{
 		switch(LPCTL(slot))
@@ -1216,14 +1221,14 @@
 		sample>>=SHIFT;
 	}
 
-	if(!STWINH(slot))
-		*RBUFDST=sample;
-
 	if(slot->EG.state==ATTACK)
 		sample=(sample*EG_Update(slot))>>SHIFT;
 	else
 		sample=(sample*EG_TABLE[EG_Update(slot)>>(SHIFT-10)])>>SHIFT;
 
+	if(!STWINH(slot))
+		*RBUFDST=sample;
+		
 	return sample;
 }
 
@@ -1243,19 +1248,18 @@
 
 		for(sl=0;sl<32;++sl)
 		{
+			RBUFDST=SCSP->RINGBUF+SCSP->BUFPTR;
 			if(SCSP->Slots[sl].active)
 			{
 				struct _SLOT *slot=SCSP->Slots+sl;
 				unsigned short Enc;
 				signed int sample;
 
-				RBUFDST=SCSP->RINGBUF+SCSP->BUFPTR;
 				sample=SCSP_UpdateSlot(SCSP, slot);
-				++SCSP->BUFPTR;
-				SCSP->BUFPTR&=63;
+
 #ifdef USEDSP
 				Enc=((TL(slot))<<0x0)|((IMXL(slot))<<0xd);
-				SCSPDSP_SetSample(&SCSP->DSP,(sample*SCSP->LPANTABLE[Enc])>>SHIFT,ISEL(slot),IMXL(slot));
+				SCSPDSP_SetSample(&SCSP->DSP,(sample*SCSP->LPANTABLE[Enc])>>(SHIFT-2),ISEL(slot),IMXL(slot));
 #endif
 				Enc=((TL(slot))<<0x0)|((DIPAN(slot))<<0x8)|((DISDL(slot))<<0xd);
 				{
@@ -1263,7 +1267,8 @@
 					smpr+=(sample*SCSP->RPANTABLE[Enc])>>SHIFT;
 				}
 			}
-
+			--SCSP->BUFPTR;
+			SCSP->BUFPTR&=63;
 		}
 
 		SCSPDSP_Step(&SCSP->DSP);

Changes:
-Corrected handling of the FM ring-buffer pointer - MDXSL and MDYSL now handled correctly
-Incorporated MDL into modulation code (not yet handled correctly, but it's something to start with)
-Changed frequency-step calculation to use UINT64 to fix rollover problems caused by large OCT values
-Improved DSP mix (I believe it's correct now)
-Fixed a minor LPSLNK problem