Using state object to capture RAW image buffer

Started by novovaccum, July 08, 2023, 09:11:48 PM

Previous topic - Next topic

0 Members and 1 Guest are viewing this topic.

novovaccum

Hello,

I'm wrapping my head around, trying to figure out how I could access RAW and JPEG buffer during image acquisition...

I could get logs from SCSS states when a picture is being taken, following this topic : https://www.magiclantern.fm/forum/index.php?topic=1915.0

I was able to output the memory addresses when CreateStateObject returns, using the qemu logs for my 50D, also following : https://www.magiclantern.fm/forum/index.php?topic=17969.msg196010#msg196010

Then I decided to create a module to log trace the different states and transitions during the image aquisition, simply reproducing what seemed to by done in the shootspy.c example. I don't know if it was a good idea, but as I was not able to find that code from ML sources (neither on `dm-spy-experiments` branch), or maybe did I miss something...?


#include <string.h>
#include <dryos.h>
#include <module.h>
#include <property.h>
#include <menu.h>
#include <config.h>
#include <console.h>
#include "../trace/trace.h"
#include "state-object.h"

/* uncomment for live debug messages */
// #define trace_write(trace, fmt, ...) { printf(fmt, ## __VA_ARGS__); printf("\n"); msleep(500); }

#define STR_APPEND(orig,fmt,...) ({ int _len = strlen(orig); snprintf(orig + _len, sizeof(orig) - _len, fmt, ## __VA_ARGS__); });

#define LOG_SIZE 10000
static char logs[LOG_SIZE] = "";

/* to start tracing while module is enabled */
static uint32_t tranlogs_trace_ctx = TRACE_ERROR;

static CONFIG_INT("tranlogs.enabled", tranlogs_enabled, 0);

// https://www.magiclantern.fm/forum/index.php?topic=1915.0

/**
************************     
********** 50D *********
************************
*/
#define SCS_STATE (*(struct state_object **)0x320C)
#define SCSES_STATE (*(struct state_object **)0x3210)
#define SCSSR_STATE (*(struct state_object **)0x3214)
#define SBS_STATE (*(struct state_object **)0x3268)
#define SDS_REAR_STATE (*(struct state_object **)0x36E0)
#define SDS_FRONT1_STATE (*(struct state_object **)0x3750)

// #define SDS_FRONT2_STATE (*(struct state_object **)0x36B4) // not found
// #define SDS_FRONT3_STATE (*(struct state_object **)0x36B8) // not found
// #define SDS_FRONT4_STATE (*(struct state_object **)0x36BC) // not found

#define SPS_STATE (*(struct state_object **)0x32B0)
#define FSS_STATE (*(struct state_object **)0x3C94)

// #define FCS_STATE (*(struct state_object **)0x3c34) // not found

static int (*StateTransition)(void*,int,int,int,int) = 0;

static int counter = 0;

static int stateobj_spy(struct state_object * self, int x, int input, int z, int t)
{   
    int old_state = self->current_state;
    int ans = StateTransition(self, x, input, z, t);
    int new_state = self->current_state;

    if (self == SBS_STATE) { STR_APPEND(logs, "SBS "); }
    else if (self == SCS_STATE) { STR_APPEND(logs, "SCS  :"); }
    else if (self == SCSES_STATE) { STR_APPEND(logs, "SCSes:"); }
    else if (self == SCSSR_STATE) { STR_APPEND(logs, "SCSsr:"); }
    else if (self == SDS_REAR_STATE) { STR_APPEND(logs, "SDSr :"); }
    else if (self == SDS_FRONT1_STATE) { STR_APPEND(logs, "SDSf1:"); }
    // else if (self == SDS_FRONT2_STATE) { STR_APPEND(logs, "SDSf2:"); }
    // else if (self == SDS_FRONT3_STATE) { STR_APPEND(logs, "SDSf3:"); }
    // else if (self == SDS_FRONT4_STATE) { STR_APPEND(logs, "SDSf4:"); }
    else if (self == SPS_STATE) { STR_APPEND(logs, "SPS  :"); }
    else if (self == FSS_STATE) { STR_APPEND(logs, "FSS  :"); }
    // else if (self == FCS_STATE) { STR_APPEND(logs, "FCS  :"); }

    STR_APPEND(logs, "(%d) -- %2d -->(%d)\n", old_state, input, new_state);
    trace_write(tranlogs_trace_ctx, "%s", logs);
   
    return ans;
}

static int stateobj_start_spy(struct state_object * stateobj)
{
    if (!StateTransition)
        StateTransition = (void *)stateobj->StateTransition_maybe;
   
    else if ((void*)StateTransition != (void*)stateobj->StateTransition_maybe) // make sure all states use the same transition function
    {
        beep();
        return;
    }
  stateobj->StateTransition_maybe = (void *)stateobj_spy;
  return 0; //not used currently
}

static void shootspy_init(void* unused)
{
    logs[0] = 0;
    stateobj_start_spy(SCS_STATE);
    stateobj_start_spy(SCSES_STATE);
    stateobj_start_spy(SCSSR_STATE);
    stateobj_start_spy(SBS_STATE);
    stateobj_start_spy(SDS_REAR_STATE);
    stateobj_start_spy(SDS_FRONT1_STATE);
    // stateobj_start_spy(SDS_FRONT2_STATE);
    // stateobj_start_spy(SDS_FRONT3_STATE);
    // stateobj_start_spy(SDS_FRONT4_STATE);
    stateobj_start_spy(SPS_STATE);
    stateobj_start_spy(FSS_STATE);
    // stateobj_start_spy(FCS_STATE);
}

static struct menu_entry tranlogs_menus[] =
{
    {
        .name           = "Transition Logs Module",
        .select         =  menu_open_submenu,
        .help           = "Log State Machine transitions",
        .max            = 1,
        .submenu_width  = 710,
        .children       = (struct menu_entry[]) {
            {
                .name   = "Enable module",
                .max    = 1,
                .priv   =  &tranlogs_enabled,
                .help   = "Will be generated once camera restart"
            },
            MENU_EOL,
        },
    },
};


static unsigned int tranlogs_init()
{       
    char filename[32] = "STATES.TXT";
       
    if(1)
    {
        tranlogs_trace_ctx = trace_start("debug", filename);
        trace_set_flushrate(tranlogs_trace_ctx, 1000);
        trace_write(tranlogs_trace_ctx, "tranlogs module: Starting trace");
    }

    menu_add("Debug", tranlogs_menus, COUNT(tranlogs_menus));

    // main task during shoot
    task_create("shootspy_init", 0x1A, 0x1000, shootspy_init, (void*)0);
   
    return 0;
}

static unsigned int tranlogs_deinit()
{
    return 0;
}

MODULE_INFO_START()
    MODULE_INIT(tranlogs_init)
    MODULE_DEINIT(tranlogs_deinit)
MODULE_INFO_END()

MODULE_CONFIGS_START()
    MODULE_CONFIG(tranlogs_enabled)
MODULE_CONFIGS_END()


Then I could retrieve the following logs from the 50D :


FSS  :(0) --  7 -->(0)
SCS  :(1) --  1 -->(2)
SCS  :(2) --  2 -->(4)
SCS  :(4) --  3 -->(5)
SCS  :(5) --  4 -->(6)
SCSes:(0) --  5 -->(1)
FSS  :(0) --  8 -->(0)
SCSes:(1) -- 14 -->(2)
SCSes:(2) -- 15 -->(4)
SCS  :(7) --  6 -->(8)


I find it a bit difficult to figured out what is going on there. Is there some debug mode I should have enabled to access more detailed logs or is there some better way to retrieve more information on those transitions?

I must admit I'm fairly new to ML and it's code. I spent some fair amount of time reading through the forum and I could not find more information on the topic. If I've missed any information, please let me know.

Thank you for your precious help



names_are_hard

Hopefully I'm wrong but I don't think anyone will know the answers to these highly specific questions.  You also don't explain what you're trying to achieve, so it's hard to give good advice.

Either way, messing about trying to understand the state machine is probably a waste of time.  ML can already capture raw data, and save it, so why not take advantage of this existing code?  See e.g. modules/silent/silent.c, silent_pic_take_fullres(), which does lots of low-level stuff to take a picture in an unusual way (without shutter activation).  You don't care about the shutter stuff I assume but perhaps this may work as a useful example of the general process.  There are other situations where we trigger image capture, so there are lots of places in the code to compare.

novovaccum

Thank you for your prompt reply!

Absolutely, I should have been more precise on the main purpose.
I'm still trying to figure out if their is a way to modify (encrypt) the full image buffer before it is written on the card.
I know io_crypt module is using hooks to replace the read/write routines to encrypt each data chunk before image data is saved.
So I was trying to see if there would be some other way to achieve the encryption.

I assumed the silent module and the silent_pic_take_fullres() to provide raw data to be saved as DNG files without the shutter activation. but because it is outside of the usual way of taking a picture on the cam, which is why I started to investigate around the state object. And also going through the Academic Corner of the forum, I found this paper : https://www.sciencedirect.com/science/article/abs/pii/S2666281721001219 where they actually found a way to modify the image buffer at a certain transition when a picture was being taken to modify its content.

But maybe it's not the way to do it? Do you think there might be some other way to achieve such a thing? With normal shutter activation?

Thank you so much for your help and advices

names_are_hard

Ah, okay.  Why do you want a different method to io_crypt?  Hooking the write function is one obvious and sensible way of doing it, and will mean all data hitting the disk is encrypted.

silent_pic_take_fullres() was an example: it demonstrates how to take a picture, and has good comments.  At one stage, the data is available for direct access / modification.  You don't have to disable the shutter if you don't want to.

novovaccum

io_crypt hooks on I/O is a really good way of doing things, despite you still have to find the specific memory addresses of the functions table and make your way into the ROM for each cam... this is why I wanted to investigate further to provide something which might be independant from a specific model.

I did try to tweak and play from the sources of silent.mo. The thing is that I obviously can modify the source buffer from the image, but still I need to know when the full shutter action was triggered, unless it can't be modified at the right moment (state transition).

Correct me if I'm wrong, currently :
silent_pic_polling_cbr will periodically check on the get_halfshutter_pressed events to trigger silent_pic_take which in turn might trigger silent_pic_take_fullres if silent_pic_mode was set for full res.

How can I manage to figure out when full shutter events is triggered to check on raw buffer?

From module.h there's crb_keypress, where I could get module_key_press_fullshutter for instance. but maybe it's not the way to do such a thing as all my tests would simply freeze everything. I don't think I get what goes behind with this cbr_keypress and if it suited to do such a thing...

Sorry if my reply doesn't make sense.


names_are_hard

Quote from: novovaccum on July 09, 2023, 07:47:50 PM
io_crypt hooks on I/O is a really good way of doing things, despite you still have to find the specific memory addresses of the functions table and make your way into the ROM for each cam... this is why I wanted to investigate further to provide something which might be independant from a specific model.

Do you have a requirement that this is fully portable code, not specific to any model?  That's going to make things massively more difficult.  I would guess you are not trying to do this.

Finding model specific values is fairly easy.  What cam are you targeting?

State machines will also have differences between models - and, most importantly of all, any technique you develop will require testing on some specific cam.  Does it matter much which route you choose when you're going to have to test it finally on a real cam?

Quote
Correct me if I'm wrong, currently :
silent_pic_polling_cbr will periodically check on the get_halfshutter_pressed events to trigger silent_pic_take which in turn might trigger silent_pic_take_fullres if silent_pic_mode was set for full res.

How can I manage to figure out when full shutter events is triggered to check on raw buffer?

I don't know, I didn't look at it in detail.  It looked like it had useful comments to work from, and it's clearly in the area that you're interested in, and works, without using state objects.  I thought you might find it a useful example to work from, that's all.

I'm also not saying the state machine approach is wrong, I don't have a good way of estimating the difficulty compared to other approaches.  Perhaps the state machine approach is easiest.  But I don't know, and I don't know how to do it that way.  io_crypt already does what you've *said* you want - adapting it to meet the goals you haven't told us sounds like the easiest route to me.  But I don't know what those goals are.

novovaccum

Quote
Do you have a requirement that this is fully portable code, not specific to any model? 

No, I absolutely have no requirements to have a fully portable code.
And since I see your answer, I just realized that my assumptions on the subject were terribly wrong!...   :'( :D

Quote
State machines will also have differences between models

Of course that each camera model will produce different state transitions!  :)  I should have paid more attention to this aspect from the start.
And that will make things even worse and massively more difficult, as you said...
Now it is clear to me, why it is fairly more elegant to seek and set values for a specific model using I/O hooks (as implemented in io_crypt), than trying to implement some logic using state transitions for different cams which will generate much more complicated code....

Especially since, doing encryption on data chunks or on full buffer image should take almost the same amount of time at the end!

Quote
What cam are you targeting?

I actually own 50D, 600D and 5D2.
From module source code 600D addresses are already set and I did some tests encrypting images which works really fine.
50D is commented and 5D2 is not defined yet. A good place for me to find those addresses from their respective ROMs.

I will let you know if I somehow get stuck, but the (commented) explanations from the source code seem pretty clear to me.

Thank you so much for your help, once again!  :)

names_are_hard

The state machine route *might* be easier...  but you have to do 100% of the work.  The io_crypt route, you only need to do 5%, everything else is already written!

All the cams you have emulate well in qemu, and I would guess you can test io_crypt there.  If I'm correct you can test in a low risk fashion.

novovaccum

Hello,

I'm trying to set iodev_table from io_crypt for my 50D and 5D2.
Following g3gg0 instructions I was able to identify the adresses but the tests on qemu make infinite loops on iodev_OpenFile :


io_crypt: Detected 50D
iodev_OpenFile('/ML/DATA/io_crypt.pub', 0) = 0
iodev_OpenFile('/ML/SETTINGS/MENU.CFG', 0) = -1
iodev_OpenFile('//ML/DATA/io_crypt.pub', 0) = 1
iodev_OpenFile('///ML/DATA/io_crypt.pub', 0) = 2
iodev_OpenFile('////ML/DATA/io_crypt.pub', 0) = 3
iodev_OpenFile('/////ML/DATA/io_crypt.pub', 0) = 4
iodev_OpenFile('//////ML/DATA/io_crypt.pub', 0) = 5
iodev_OpenFile('///////ML/DATA/io_crypt.pub', 0) = 6
iodev_OpenFile('////////ML/DATA/io_crypt.pub', 0) = 7
iodev_OpenFile('/////////ML/DATA/io_crypt.pub', 0) = 8
 

The adresses of iodev_table I used are 0x1F208 for the 50D and 0x22568 for the 5D2.

Following the instructions, 50D's tables of function references to 0x1F200, then second entry in the table should be 0x1F208. Same for the 5D2 where its tables of function references to 0x22560. Or did I misinterpreted?

Looking at the 600D (already defined in the code) first table entry is 0x1E67C, then the second is 0x1E684 which is working.

What could be the issue there?






names_are_hard

Presumably, it shouldn't keep adding another / to the path each time.  I'd want to debug the code to see why this is occuring.  You can attach gdb to qemu.

novovaccum

I did check the logs from qemu to compare 600D and 50D. Doing trace on calls and looking at iodev_OpenFile hook, for what I understand, it seems that the "flags" param is set to 0 in 50D logs. And file descriptor param also does not seem accurate.

50D logs :

call 0x8D69C(1adb10 "iodev_OpenFile('/ML/DATA/io_crypt.key', 0) = 0", 1, 0, 1add34 &"/ML/DATA/io_crypt.key")
                                                                                 at [module_task:8d974:94b434]
             call 0x8D5CC(9f1ce, 1adb10 "iodev_OpenFile('/ML/DATA/io_crypt.key', 0) = 0", 0, 1add34 &"/ML/DATA/io_crypt.key")
                                                                                 at [module_task:8d6b0:8d978]
              call 0xFF98E1E4(1ada48, 7f, 9f1ce, 1adad4 &"iodev_OpenFile('/ML/DATA/io_crypt.key', 0) = 0")
                                                                                 at [module_task:8d5ec:8d6b4]
              return 2e to 0x8D5F0                                               at [module_task:ff98e594:8d6b4]
             return 0 to 0x8D6B4                                                 at [module_task:8d614:8d978]
             call 0x99C34(0, 690, c0123000, 1adb10 "iodev_OpenFile('/ML/DATA/io_crypt.key', 0) = 0")
                                                                                 at [module_task:8d89c:8d978]
              call 0x99B14(0, 690, c0123000, 1adb10 "iodev_OpenFile('/ML/DATA/io_crypt.key', 0) = 0")
                                                                                 at [module_task:99c40:8d8a0]


600D logs :

call 0xCC4E3C(194c28 "iodev_OpenFile('/ML/DATA/io_crypt.key', 69632) = 0", 1, 0, 0)
                                                                                 at [module_task:cc5114:bbc8f4]
            call 0xCC4D6C(cd6c59, 194c28 "iodev_OpenFile('/ML/DATA/io_crypt.key', 69632) = 0", 0, 0)
                                                                                 at [module_task:cc4e50:cc5118]
             call 0xFF1F94BC(194b60, 7f, cd6c59, 194bec &"iodev_OpenFile('/ML/DATA/io_crypt.key', 69632) = 0")
                                                                                 at [module_task:cc4d8c:cc4e54]
             return 32 to 0xCC4D90                                               at [module_task:ff1f9868:cc4e54]
            return 0 to 0xCC4E54                                                 at [module_task:cc4db4:cc5118]
            call 0xCD1674(230, 690, c0123000, 194c28 "iodev_OpenFile('/ML/DATA/io_crypt.key', 69632) = 0")
                                                                                 at [module_task:cc503c:cc5118]
             call 0xCD1554(230, 690, c0123000, 194c28 "iodev_OpenFile('/ML/DATA/io_crypt.key', 69632) = 0")
                                                                                 at [module_task:cd1680:cc5040]
             return 0 to 0xCD1684                                                at [module_task:cd161c:cc5040]


Could that be related to all the subsequent calls to open?
What could cause the params to be inaccurate?
What should I investigate from there?

Thank you again for your help

names_are_hard

I don't know the iocrypt code, I don't know what code you've changed, and I don't know if the addresses you've found are valid.  I don't know what the params do in iodev_OpenFile().  Anything could be going on.  I'd need to spend quite a lot of time to try to investigate this problem, and I don't really want to.  It's not really ML related, and besides, this is for your Uni course, right?

I don't really know what those logs show, I've never worked with iocrypt.  I agree it looks like the two cams are doing very different things, and they probably shouldn't look this different, but that's just a guess.

Quote
What could cause the params to be inaccurate?
Lots of things - hence my suggestion to use a debugger to find out what is going on inside qemu.  You know the code that's being run, and presumably you have some idea of what you want to get passed to various functions.  This is a standard debugging problem.  Work forwards from where you think things are still good, work backwards from where you know they're bad, and try and work out what causes the change.

novovaccum

Quote
I don't know what code you've changed

The only thing that has been added to the code is the iodev_table address for the 50D and 5D2, which I found using GDB logging hooks.

Quote
I'd need to spend quite a lot of time to try to investigate this problem, and I don't really want to.

I fully understand your position and I'm not waiting on you to find my problem here :) However, this problem could be ML related for someone who has a 50D or 5D2 and would like to use io_crypt.

I don't think debugging before and after these calls to see what is passed to iodev_OpenFile or other functions will give me a better understanding of the cause of the problem and what could be changed to solve it, as I don't have a deep understanding of the design and implementation of this part of io_crypt at this level.

My assumptions are that the iodev_table addresses I found are not correct. I'll try to investigate further and if I'm still stuck, I might stick to my 600D which works very well for the purpose.

Thanks again for all your answers

names_are_hard

Well, debugging is hard - but it leads to increased understanding, that's the point!  If you want to fix buggy code, you have to understand it :)

I checked the history for io_crypt, and I see that it used to be enabled on 50D.  It was disabled with this commit: https://github.com/reticulatedpines/magiclantern_hg_02/commit/a4e4ee71010c320e2bb5e54c87f8eca232473285
"io_crypt: disabled models where reading was reported to be buggy".  I didn't have any luck trying to find those reports, but didn't try for long.

So I think it's expected not to work on 50D (at present).  If you can find the reports maybe you'll get more info.

novovaccum

Quote
Well, debugging is hard - but it leads to increased understanding, that's the point!  If you want to fix buggy code, you have to understand it :)

Absolutely! It increases understanding I agree :) My point was more related to the use of iodev_table in this context which I have no knowledge of.  :'( So I don't think I would have been able to fix that issue there...

Quote
So I think it's expected not to work on 50D (at present).  If you can find the reports maybe you'll get more info.

Thank you for putting back that information.  :D I went trough the other commits. No more information about the 50D except its configuration was simply disabled after that commit : https://foss.heptapod.net/magic-lantern/magic-lantern/-/commit/456e79550daa919e8eb3c1f571c4906679045616
I also went back to look at io_crypt related topic : https://www.magiclantern.fm/forum/index.php?topic=10279.50
It seems that for certain model there were some recusive call to open until file descritors run out.... presumably the behaviour I encountered with the 50D.

I think I'll simply move forward and will use the 600D 

Thank you again  :)

names_are_hard

Quote
the use of iodev_table in this context which I have no knowledge of.  :'( So I don't think I would have been able to fix that issue there

Yes, if you understood everything, you wouldn't have any difficulties fixing the bug :)  That's what you'd be using gdb for - to determine what's happening, how the parts you don't understand are used, and then you can change whatever is wrong so that things work.

This kind of investigation is normal when porting code to different model cameras, and very likely already happened many times to get io_crypt to support the cams that it does.  The majority of software dev time isn't writing code, it's fixing bugs.  Even more so when you're working on an undocumented system!

Nothing wrong with using the easy cam for the work, if it still meets the requirements of the project :)

novovaccum

Quote
The majority of software dev time isn't writing code, it's fixing bugs.  Even more so when you're working on an undocumented system!

In this context, you're absolutely right! I'm now starting to get your point, since I got my hands (a little bit :D) into the process. And all your answers give me a clearer view of the challenges of porting things to different cameras. It also makes me realize how much work it can be for someone new to the project. But I'm still willing to keep digging for further improvements.  :)

Still it's a long way to go through it. I mean, it took me quite some time just to get some initial setup having qemu working and figuring out how things are nested in the code.  :)

As you say, this is the state of things working in an undocumented project. But the forum is a very helpful resource for that. There's a lot of information available as long as you take the time for it. Thank you very much for taking the time to put your experience and knowledge here!  :)