UHS-I / SD cards investigation

[RawCutz]

Hi guys I totally appreciate what y'all are doing.

@theBilalFakhouri is it possible for you to compile a build for my 700D with the latest sd_uhs developments?

I have been using an old build compiled by you for some time now but it struggles with 14bit @ 1736x972 unless I under expose.

I used on old build from @Danne as well and it worked just fine but the same issue.

I have the Sandisk Extreme Pro 64GB 170mb/s.
 

[RawCutz]

@theBilalFakhouri

Your build is the only one that have worked with sd_uhs.

I have never had any success with the builds from Danne and Dfort.

Most likely because I dont know how to compile.

[Danne]

Maybe we should look into:
https://bitbucket.org/hudson/magic-lantern/src/sd_uhs/modules/sd_uhs/sd_uhs.c#lines-59
drv_strength

Also note, if this isn´t applied:

Code: [Select]

    /* enable SDR104 */
    //patch_hook_function(sd_set_function, MEM(sd_set_function), sd_set_function_log, "SDR104");
    //SD_ReConfiguration();
We can skip this:

Code: [Select]

//static void sd_set_function_log(uint32_t* regs, uint32_t* stack, uint32_t pc)
//{
  //  qprintf("sd_set_function(0x%x)\n", regs[0]);
   
    /* UHS-I SDR50? */
    //if (regs[0] == 0xff0002)
    //{
        /* force UHS-I SDR104 */
      //  regs[0] = 0xff0003;
   // }
// }
EDIT:
Also, basically what is needed for the eosm is this(reduced the code):

Code: [Select]

/**
 * Experimental SD UHS overclocking.
 */

#include <module.h>
#include <dryos.h>
#include <patch.h>
#include <console.h>
#include <config.h>
#include <lens.h>

/* camera-specific parameters */
static uint32_t sd_setup_mode = 0;
static uint32_t sd_setup_mode_in = 0;
static uint32_t sd_setup_mode_reg = 0xFFFFFFFF;
static uint32_t sd_set_function = 0;

static uint32_t uhs_regs[]     = { 0xC0400600, 0xC0400604,/*C0400608, C040060C*/0xC0400610, 0xC0400614, 0xC0400618, 0xC0400624, 0xC0400628, 0xC040061C, 0xC0400620 };   /* register addresses */
static uint32_t sdr_160MHz[]   = {        0x2,        0x2,                             0x1, 0x1D000001,        0x0,      0x100,      0x100,      0x100,        0x1 };   /* overclocked values: 160MHz = 96*(4+1)/(2?+1) (found by brute-forcing) */
static uint32_t sdr_192MHz[]   = {        0x8,        0x3,                             0x4, 0x1D000301,        0x0,      0x201,      0x201,      0x100,        0x4 };
static uint32_t sdr_240MHz[]   = {        0x8,        0x3,                             0x3, 0x1D000301,        0x0,      0x201,      0x201,      0x100,        0x3 };

static uint32_t uhs_vals[COUNT(uhs_regs)];  /* current values */
static int sd_setup_mode_enable = 0;
static int turned_on = 0;
static CONFIG_INT("sd.sd_overclock", sd_overclock, 2);

/* start of the function */
static void sd_setup_mode_log(uint32_t* regs, uint32_t* stack, uint32_t pc)
{
    qprintf("sd_setup_mode(dev=%x)\n", regs[0]);
   
    /* this function is also used for other interfaces, such as serial flash */
    /* only enable overriding when called with dev=1 */
    sd_setup_mode_enable = (regs[0] == 1);
}

/* called right before the case switch in sd_setup_mode (not at the start of the function!) */
static void sd_setup_mode_in_log(uint32_t* regs, uint32_t* stack, uint32_t pc)
{
    qprintf("sd_setup_mode switch(mode=%x) en=%d\n", regs[sd_setup_mode_reg], sd_setup_mode_enable);
   
    if (sd_setup_mode_enable && regs[sd_setup_mode_reg] == 4)   /* SDR50? */
    {
        /* set our register overrides */
        for (int i = 0; i < COUNT(uhs_regs); i++)
        {
            MEM(uhs_regs[i]) = uhs_vals[i];
        }
       
        /* set some invalid mode to bypass the case switch
         * and keep our register values only */
        regs[sd_setup_mode_reg] = 0x13;
    }
}

static void sd_overclock_task()
{
   
    /* install the hack */
    if (sd_overclock == 1) memcpy(uhs_vals, sdr_160MHz, sizeof(uhs_vals));
    if (sd_overclock == 2) memcpy(uhs_vals, sdr_192MHz, sizeof(uhs_vals));
    if (sd_overclock == 3) memcpy(uhs_vals, sdr_240MHz, sizeof(uhs_vals));
    patch_hook_function(sd_setup_mode, MEM(sd_setup_mode), sd_setup_mode_log, "SD UHS");
    patch_hook_function(sd_setup_mode_in, MEM(sd_setup_mode_in), sd_setup_mode_in_log, "SD UHS");

}

static struct menu_entry sd_uhs_menu[] =
{
    {
        .name   = "sd overclock",
        .priv   = &sd_overclock,
        .max    = 3,
        .choices = CHOICES("OFF", "160MHz", "192MHz", "240MHz(feeling lucky)"),
        .help   = "Select a patch and restart camera. Disable with OFF and restart",
        .help2  = "Proven working with 95Mb/s and 170Mb/s cards",
    }
};

static struct menu_entry sd_uhs_menu1[] =
{
    {
        .name   = "sd overclock",
        .priv   = &sd_overclock,
        .max    = 3,
        .choices = CHOICES("OFF", "160MHz", "192MHz", "240MHz (feeling lucky)"),
        .help   = "Select a patch and restart camera. Disable with OFF and restart",
        .help2  = "Proven working with 95Mb/s and 170Mb/s cards",
    }
};

static unsigned int sd_uhs_init()
{
   
    //needed with manual lenses cause it stalls liveview. Maybe helps for cams like 6D. To be tested.
    while (is_movie_mode() && !lv)
    {
        msleep(100);
    }
   
    if (is_camera("EOSM", "2.0.2"))
    {
        sd_setup_mode       = 0xFF338D40;
        sd_setup_mode_in    = 0xFF338DC8;
        sd_setup_mode_reg   = 1;
        sd_set_function     = 0xFF63EF60;
        if (sd_overclock)
        {
            sd_overclock_task();
            turned_on = 1;
        }
    }
   
    menu_add("Movie", sd_uhs_menu, COUNT(sd_uhs_menu));
    menu_add("Debug", sd_uhs_menu1, COUNT(sd_uhs_menu1));
   
    return 0;
}

static unsigned int sd_uhs_deinit()
{
    return 0;
}

MODULE_INFO_START()
MODULE_INIT(sd_uhs_init)
MODULE_DEINIT(sd_uhs_deinit)
MODULE_INFO_END()

MODULE_CONFIGS_START()
MODULE_CONFIG(sd_overclock)
MODULE_CONFIGS_END()

[Bender@arsch]

Because of the 131mb/s writespeed limit, I tested just for fun the full resolution mode (5d3, 5784x3254 7,4fps at 10bit) with card spanning and the 160mhz module.

writspeed:
CF: 105MB/s, 42% idle
SD: 65MB/s, 12% idle

=170MB/s with green icon

[Danne]

Hehe, not bad! Reduced fps maximizes card speed.

[70MM13]

doesn't this point to cpu overhead within the magic lantern code?
slower fps reduces the burden on the cpu, no?
it would be great to know where the most cpu resources are going during usage.  is there a way to measure this, akin to "process explorer" in windows that shows exactly how much cpu resources are being used by each running process?

[Bender@arsch]

OK I think this is a failure:
3840x1440 x 23,976fps = ~130 mega pixel
5784x3254 x 7,4fps = ~140 mega pixel

I need ~125mb/s for 4K at 10bit, so 170mb/s is not correct I think.

[70MM13]

hopefully the write speed indicator is a true readout of the rate of data being written to the cards.

[Bender@arsch]

OK, I checked it out:
~1600mb (cf 830, sd 768), 102
frames =~14sec
1600/14 =114mb/s

[70MM13]

can you try the same comparison between rates that are reported versus recorded with both full sensor recording and cropped modes?  let's see if the same discrepancy exists...

[Bender@arsch]

Ok. Here are some short tests:

Full res. 5784x3254 7,4fps 10bit
CF: 105 42% idle
SD: 65 12%idle
=170MB/s
-> 1600 (cf 830, sd 768), 102 frames =~14sec
1600/14 =114mb/s

4K 3840x1536 23,976fps 10bit
CF: 70,4, 2% idle
SD: 50,4 6% idle
=120,8MB/s
-> 2791 (cf 1619, sd 1172), 558 frames =~23sec
2791/23 =121,3mb/s

4K half 4096x2304 11,988fps 10bit
CF: 108   22% idle
SD: 61,5 80% idle
=169,5MB/s
-> 1531 (cf 1340, sd 197), 191 frames =~16sec
1531/16 =95,6mb/s

Anamorphic 1808x2300 23,976fps 10bit
CF: 93 30% idle
SD: 53 66% idle
=146MB/s
-> 1397 (cf 1039, sd 358), 401 frames =~17sec
1397/17 =82,17mb/s

Anamorphic 1808x2300 23,976fps 12bit
CF: 90 30% idle
SD: 53 50% idle
=143MB/s
-> 1271 (cf 910, sd 361), 334 frames =~14sec
1271/14 =90,7mb/s

Anamorphic 1808x2300 23,976fps 14bit
CF: 77 15% idle
SD: 53 20% idle
=130MB/s
-> 1714 (cf 991, sd 723), 388 frames =~16sec
1714/16 =107mb/s

[70MM13]

well, isn't that interesting!

looks like we need to stick to only one preset for accurate speed reporting during recording...

hopefully someone familiar with the codebase can figure out what's going on!

nice catch...

[rinski]

Full anamorphic 1.1 1920x3760 14 bits 15,003 fps, 6 minutes 240 mhz with little drop in speed 91 MB / s cf lexar 1066x 74.5 MB / s, sdxc sandisk extreme pro 170 MB / s. Thanks for your work.

[theBilalFakhouri]

Canon 700D with Sandisk Extreme Pro 170 MB/s @ 240 MHz:
2560x1280 10-bit lossless @ 23.976 FPS - Continuous
2560x1440 10-bit lossless @ 23.976 FPS - Mostly Continuous (Depending on scene with Frozen Preview)

Canon 700D with Sandisk Extreme Pro 95 MB/s @ 240 MHz:
2560x1440 10-bit lossless @ 23.976 FPS  - Continuous (Sometimes with Frozen Preview)

-However 240 MHz overclock with Sandisk Extreme Pro 95 MB/s isn't stable.
-2560x1280 is 2:1 Aspect Ratio

[yourboylloyd]

I want to test on my t3i. I'm going to buy a bestbuy today to get a faster sd card. Is there a test build/module for that one?

[theBilalFakhouri]

I want to test on my t3i. I'm going to buy a bestbuy today to get a faster sd card. Is there a test build/module for that one?

Nope, t3i is a DIGIC 4 camera, many features aren't there yet for t3i, like lossless compression, higher resolutions and sd overclock, I do not recommended it for ML use with these features.

[reddeercity]

It funny that you are talking about t3i/d4  cam , last night i was reading though my 5d2 rom disassembly
Looking for CF card udma string to set udma speed (e.g. udma6 5d2/50d to umda7 (120Mb) ) and Lossless stuff
I found the SD_SoftReset etc. ... all the stuff that you guy are playing around in the D5 Cams e.g. 700D etc. ... in the d4 rom
So the max speed in the rom for SD card is 48MHz plus there  is a 24MHz mode also , not sure what speed the T3i records at ?
but I think it's doable  to apply this code to t3i's if there not at max already , to be sure t3i can't go over 48MHz unless
the bus speed is over clocked . & Hi res crop_rec  is only a matter of some one following what i did to the 50D .
It would be the same , i totally expect crop_rec to work on the t3i , but i don't have the time to do it my self (i will help thou )

Any how found a couple of string for CF

Code: [Select]

ff38b1f8: STRING:  'cfIdentifyDrive: Set UDMA( Mode=%d )'
ff38b1b0: 128f20dc addne r2, pc, #220 ; *'CF_DeviceCreate: SoftReset
ffbdbb48: STRING:  'CF_GetAccessTiming : DatTim = %d, DatMod = %d' 

I hoping to set CF to UDMA7 from 6 by code ,

I wondering are you able to set the band width when you are switch to hi-speed in the sd card on 700d ?
I see this in d4 rom

Code: [Select]

ff395544: STRING:  'SD_SetBusWidth: Width(Width=%d)'
ff395564: STRING:  'SD_SetBusWidth(CMD55)'
ff3955b0: STRING:  'SD_SetBusWidth(ACMD6)'

Also when over clocking to a higher speed dose the voltage increase to support the data output ?
Not sure how you would check that , it seems to me that same of the unstable results are some missing configuration or reg 
my 2cents worth 
Here a older (2001) Document on SD card I/O ,  nice read
http://ugweb.cs.ualberta.ca/~c274/resources/hardware/SDcards/SD_SDIO_specsv1.pdf

Great Work on this @theBilalFakhouri ,

Edit:
found this in the 700d 115 rom , i have

Code: [Select]

ff74aafc: e28f2e26 add r2, pc, #608 ; ff74ad64: (20746553)  *"Set 1.8V Signaling"
ff74d684: e28f20fc add r2, pc, #252 ; ff74d788: (65536473)  *"sdSendVoltageSwitch"
interesting ! i think the voltage for the older cards are around 3.3Vdc , or unless the newer cards use less voltage .
maybe other cam are having a voltage switching problem ? just a thought

[theBilalFakhouri]

@reddeercity

I also have the most if not all of CF stuff in my 700D ROM, probably Canon has generic sections in their cameras firmware.

Also when overclocking to a higher speed dose the voltage increase to support the data output ?

Nope, the voltage stays at 1.8 V for UHS-I cards


5D3 needs 1.8 Volt patch maybe to enable the overclocking, see sd_uhs.c .

For SD_SetBusWidth I don't know if it possible to call it I even don't know what it does, I need more Reverse Engineering and Assembly language skills in addition to C skills, Last time I tried to call sdSoftReset with Sandisk Extreme PRO 95 MB/s the card didn't boot in the camera again I thought I broke it or modified its firmware, fortunately after a quick format using SD Formatter from SD Association it worked again.

All what I did to achieve higher clock speeds is playing with the registers which control the clock speed, but reverse engineering work and figuring out how to call these registers and changing its values are done by a1ex , I am working to improve my skills to do some tasks on my own

I hoping to set CF to UDMA7 from 6 by code ,

Good Luck with this! if it technically possible it might be doable.

it seems to me that same of the unstable results are some missing configuration or reg 

Yeah that what's I am thinking too, maybe some more calls are needed.

[70MM13]

what do you all think about the 5d3 beyond 160 mhz?

160 has been rock solid, but not quite fast enough to do 3616*1536 with no sudden stops.  i'm wondering if 192 might be just fast enough!

has anyone tried faster cards than the sandisk 170mb/s? 

[Walter Schulz]

has anyone tried faster cards than the sandisk 170mb/s? 

I suppose there are no faster cards. UHS-II cards have to run in UHS-I mode and all conventional tests (without overclocking) are showing a bit lower numbers. Fastest results are with SanDisk's properietary overclocking UHS-I cards (labeled > 95 MB/s) and SanDisk cardreaders supporting overclocking protocol. See https://cameramemoryspeed.com

Examples:
Fastest UHS-I card in SanDisk reader (supporting proprietary protocol): ~100 MByte/s. But in all other cardreaders: Up to ~85 MByte/s.
Fastest UHS-I card in conventional readers: ~93 MByte/s.

[70MM13]

nice link, thanks...

i looked at the eos r test, which seemed to be the newest one:
https://www.cameramemoryspeed.com/canon-eos-r/best-sd-cards/

"The camera is also compatible with UHS-I cards and supports SDR104 (up to 104 MB/s). The fastest UHS-I cards averaged up to 75.6 MB/s write speed in the EOS R."

it was done in 2018, so the fastest uhs-1 card they tested was the 95mb/s version.  it would be interesting to see if the 170 fares any better.

[IDA_ML]

Because of the 131mb/s writespeed limit, ...

I was wondering where does this number come from?  And if the 5D3 cannot write faster, what is this SD-overclocking effort on the 5D3 all about?  Can't we get this write speed just by using Card spanning?  The 5D3 can write at 120 MB/s on the CF card and 21 MB/s on the SD card.  Adding these numbers results in 141 MB/s write speed which is still by 10 MB/s more that 131 MB/s. 

If someone could explain this to me, I should greatly appreciate that.

[70MM13]

this is a question that is largely unanswered at this time.

there's overhead involved that changes the way it performs when spanning is enabled.  immediately the CF writing is slowed down as a result.  and in actual usage, the CF rarely goes far above 90 mb/s on it's own to begin with.  once spanning is enabled, it drops from there...

in my usage of spanning and overclock combined, i find that the faster the sd is clocked, the better the results overall.  it gets closer to 50/50 writing as the sd clock goes up.  when it was briefly working at 240 mhz, recording was very robust at the highest resolution and bit depth with very close to 50/50 writing.

i think there is a bottleneck in the software loop that is resulting in the 131 mb/s total speed limit, but it is only a guess.  if anyone is interested in testing it, i would suggest disabling components in mlv_lite that are not needed for video recording (such as sound recording and anything else you can think of) to see if this holds true. it would be interesting to see the results.

maybe higher framerates would be achievable...

edit: let me rephrase that last part to avoid potential confusion: i don't mean disabling them by the user in the magic lantern menus, rather disabling them in the code and recompiling the actual module, to tighten the main loop.  that may be too general of a description, and i don't know enough about the codebase for a better guess about what can be done to tighten it.  it could possibly be only related to i/o routines, or even more specific still.

or it could simply be a hardware limitation creating the 131 mb/s limit.

i haven't done programming in 36 years ( 6809 assembly) and all this high level language stuff is not my domain.

[IDA_ML]

Thanks a lot 70MM13,

This 131 MB/s number is pretty precise.  I wonder how this number has been measured or otherwise determined.  Can anyone elaborate on that?

[70MM13]

for me it is just from experience.  no matter how high the overclocking went during that short time it was working here, the total never exceeded 131...