OpenEXRCore 3.1.1 - A detailed writing example

[arabadzhiyski]

I'm working on a function that writes EXRs using Core only and what I have below does the job except all pixels have the same value (i.e. we get a constant color image) eventually. The value in questions seems to be the the RGBA value of the upper left most pixel. I couldn't find a detailed exr write example in the documentation, so I looked at the src of the EXR tests which helped a lot to get to the function here:

void writeEXR(Image *image, char *filename) {
    int pixel_count = image->width * image->height;
    size_t total_size = pixel_count * 4 * sizeof(float);

    // PREPARE RAW IMAGE DATA
    uint8_t *data = malloc(total_size);
    memset(data, 0, total_size);

    for (int y = 0; y < image->height; y++) { // Iterate over the lines
        for (int x = 0; x < image->width; x++) { // Iterate over the line pixels
            unsigned index = (y * image->width + x) * sizeof(float);
            printf("%d ", index);
            // This should give us planar layout (e.g. 4x1 px): 
            // 0__________________   1__________________   2__________________   3__________________
            // 0___ 4___ 8___ 12__   0___ 4___ 8___ 12__   0___ 4___ 8___ 12__   0___ 4___ 8___ 12__   
            // RRRR RRRR RRRR RRRR | GGGG GGGG GGGG GGGG | BBBB BBBB BBBB BBBB | AAAA AAAA AAAA AAAA 

            memcpy(data + (pixel_count * 0 * sizeof(float) + index), &image->pixels[y][x].color.r, sizeof(float));
            memcpy(data + (pixel_count * 1 * sizeof(float) + index), &image->pixels[y][x].color.g, sizeof(float));
            memcpy(data + (pixel_count * 2 * sizeof(float) + index), &image->pixels[y][x].color.b, sizeof(float));
            memcpy(data + (pixel_count * 3 * sizeof(float) + index), &image->pixels[y][x].alpha,   sizeof(float));
        }
    }

    // INIT EXR
    exr_context_initializer_t ctxinit = EXR_DEFAULT_CONTEXT_INITIALIZER;
    ctxinit.max_image_width = image->width;
    ctxinit.max_image_height = image->height;

    exr_context_t out;
    int partidx = 0;

    exr_start_write(&out, filename, EXR_WRITE_FILE_DIRECTLY, &ctxinit);

    exr_get_count(out, &partidx);
    assert(partidx == 0);

    exr_add_part(out, "Ci", EXR_STORAGE_SCANLINE, &partidx);
    assert(partidx == 0);

    exr_initialize_required_attr_simple(out, partidx, image->width, image->height, EXR_COMPRESSION_ZIPS);
    
    exr_add_channel(out, partidx, "A", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LINEAR, 1, 1);
    exr_add_channel(out, partidx, "B", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);
    exr_add_channel(out, partidx, "G", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);
    exr_add_channel(out, partidx, "R", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);

    exr_attr_set_user(out, partidx, "user", "mytype", 4, "foo");
    exr_write_header(out);

    int nchunks = 0;
    exr_get_chunk_count(out, partidx, &nchunks);

    exr_chunk_info_t cinfo;
    exr_encode_pipeline_t encoder;

    exr_write_scanline_chunk_info(out, partidx, 0, &cinfo);
    exr_encoding_initialize(out, partidx, &cinfo, &encoder);

    encoder.channels[0].channel_name = "A";
    encoder.channels[0].encode_from_ptr = &data[pixel_count * sizeof(float) * 3];
    encoder.channels[0].user_data_type = EXR_PIXEL_FLOAT;
    encoder.channels[0].user_pixel_stride = 1 * sizeof(float);
    encoder.channels[0].user_line_stride = sizeof(float) * image->width;
    encoder.channels[0].user_bytes_per_element = sizeof(float);

    encoder.channels[1].channel_name = "B";
    encoder.channels[1].encode_from_ptr = &data[pixel_count * sizeof(float) * 2];
    encoder.channels[1].user_data_type = EXR_PIXEL_FLOAT;
    encoder.channels[1].user_pixel_stride = 1 * sizeof(float);
    encoder.channels[1].user_line_stride = sizeof(float) * image->width;
    encoder.channels[1].user_bytes_per_element = sizeof(float);

    encoder.channels[2].channel_name = "G";
    encoder.channels[2].encode_from_ptr = &data[pixel_count * sizeof(float) * 1];
    encoder.channels[2].user_data_type = EXR_PIXEL_FLOAT;
    encoder.channels[2].user_pixel_stride = 1 * sizeof(float);
    encoder.channels[2].user_line_stride = sizeof(float) * image->width;
    encoder.channels[2].user_bytes_per_element = sizeof(float);

    encoder.channels[3].channel_name = "R";
    encoder.channels[3].encode_from_ptr = &data[pixel_count * sizeof(float) * 0];
    encoder.channels[3].user_data_type = EXR_PIXEL_FLOAT;
    encoder.channels[3].user_pixel_stride = 1 * sizeof(float);
    encoder.channels[3].user_line_stride = sizeof(float) * image->width;
    encoder.channels[3].user_bytes_per_element = 4;

    exr_encoding_choose_default_routines(out, partidx, &encoder);
    
    for (int y = 0; y < nchunks; y++) {
        if (y > 0) {
            exr_write_scanline_chunk_info(out, partidx, y, &cinfo);
            exr_encoding_update(out, partidx, &cinfo, &encoder);
        }
        exr_encoding_run(out, partidx, &encoder);
    }
    
    exr_encoding_destroy(out, &encoder);
    exr_finish(&out);
}

Yet, I must be doing something wrong. Can anybody spot the problem? A nudge in the right direction would be awesome! Thanks!

[kdt3rd]

hrm, yeah, the setting up of the pointers is a bit upside down, you want use the setup for each chunk and then init /update the writing context for each chunk written. You can see a good / simple writing loop at src/test/OpenEXRCoreTest/writing.cpp line 759 or so is doEncodeScan

[kdt3rd]

the core layer does not have utilities to write a whole image, it is a bit lower level and forces you to work in chunks...

[arabadzhiyski]

Thank you so much! It worked!

I had believed that the pointers were only needed for the starting locations of the channel data and that the encoding pipeline would do the chunk striding inside channel. (I found doEncodeScan() in src/test/OpenEXRCoreTest/compression.cpp).

void writeEXR(Image *image, char *filename) {
    int pixel_count = image->width * image->height;
    size_t pixel_stride = sizeof(float);
    size_t total_size = pixel_count * 4 * pixel_stride;

    // PREPARE RAW IMAGE DATA
    uint8_t *data = malloc(total_size);
    memset(data, 0, total_size);

    for (int y = 0; y < image->height; y++) { // Iterate over the lines
        for (int x = 0; x < image->width; x++) { // Iterate over the line pixels
            unsigned index = (y * image->width + x) * pixel_stride;

            // This should give us planar layout (e.g. 4x1 px): 
            // 0__________________   1__________________   2__________________   3__________________
            // 0___ 4___ 8___ 12__   0___ 4___ 8___ 12__   0___ 4___ 8___ 12__   0___ 4___ 8___ 12__   
            // RRRR RRRR RRRR RRRR | GGGG GGGG GGGG GGGG | BBBB BBBB BBBB BBBB | AAAA AAAA AAAA AAAA 

            memcpy(data + (pixel_count * 0 * pixel_stride + index), &image->pixels[y][x].color.r, pixel_stride);
            memcpy(data + (pixel_count * 1 * pixel_stride + index), &image->pixels[y][x].color.g, pixel_stride);
            memcpy(data + (pixel_count * 2 * pixel_stride + index), &image->pixels[y][x].color.b, pixel_stride);
            memcpy(data + (pixel_count * 3 * pixel_stride + index), &image->pixels[y][x].alpha,   pixel_stride);
        }
    }

    // INIT EXR
    exr_context_initializer_t ctxinit = EXR_DEFAULT_CONTEXT_INITIALIZER;
    ctxinit.max_image_width = image->width;
    ctxinit.max_image_height = image->height;

    exr_context_t out;
    int partidx = 0;

    // OPEN FOR WRITE
    exr_start_write(&out, filename, EXR_WRITE_FILE_DIRECTLY, &ctxinit);

    exr_get_count(out, &partidx);
    assert(partidx == 0);

    exr_add_part(out, "Ci", EXR_STORAGE_SCANLINE, &partidx);
    assert(partidx == 0);

    exr_initialize_required_attr_simple(out, partidx, image->width, image->height, EXR_COMPRESSION_RLE);
    
    exr_add_channel(out, partidx, "A", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LINEAR,      1, 1);
    exr_add_channel(out, partidx, "B", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);
    exr_add_channel(out, partidx, "G", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);
    exr_add_channel(out, partidx, "R", EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);
    
    exr_attr_set_user(out, partidx, "user", "mytype", 4, "foo");
    exr_write_header(out);

    int nchunks = 0;
    exr_get_chunk_count(out, partidx, &nchunks);

    int lines_per_chink = 0;
    exr_get_scanlines_per_chunk(out, partidx, &lines_per_chink);

    size_t line_stride = pixel_stride * image->width;
    size_t chunk_stride = line_stride * lines_per_chink;

    exr_chunk_info_t cinfo;
    exr_encode_pipeline_t encoder;

    bool first = true;
    
    for (int y = 0; y < nchunks; y++) {
        
        exr_write_scanline_chunk_info(out, partidx, y, &cinfo);
        
        if (first) {
            exr_encoding_initialize(out, partidx, &cinfo, &encoder);
        }
        else {
            exr_encoding_update(out, partidx, &cinfo, &encoder);
        }
        
        encoder.channels[0].encode_from_ptr = &data[(pixel_count * pixel_stride * 3) + chunk_stride * y];
        encoder.channels[0].channel_name = "A";
        encoder.channels[0].user_data_type = EXR_PIXEL_FLOAT;
        encoder.channels[0].user_pixel_stride = pixel_stride;
        encoder.channels[0].user_line_stride = line_stride;
        //encoder.channels[0].user_bytes_per_element = pixel_stride;

        encoder.channels[1].encode_from_ptr = &data[(pixel_count * pixel_stride * 2) + chunk_stride * y];
        encoder.channels[1].channel_name = "B";
        encoder.channels[1].user_data_type = EXR_PIXEL_FLOAT;
        encoder.channels[1].user_pixel_stride = pixel_stride;
        encoder.channels[1].user_line_stride = line_stride;
        //encoder.channels[1].user_bytes_per_element = pixel_stride;

        encoder.channels[2].encode_from_ptr = &data[(pixel_count * pixel_stride * 1) + chunk_stride * y];
        encoder.channels[2].channel_name = "G";
        encoder.channels[2].user_data_type = EXR_PIXEL_FLOAT;
        encoder.channels[2].user_pixel_stride = pixel_stride;
        encoder.channels[2].user_line_stride = line_stride;
        //encoder.channels[2].user_bytes_per_element = pixel_stride;

        encoder.channels[3].encode_from_ptr = &data[(pixel_count * pixel_stride * 0) + chunk_stride * y];
        encoder.channels[3].channel_name = "R";
        encoder.channels[3].user_data_type = EXR_PIXEL_FLOAT;
        encoder.channels[3].user_pixel_stride = pixel_stride;
        encoder.channels[3].user_line_stride = line_stride;
        //encoder.channels[3].user_bytes_per_element = pixel_stride;

        if (first) {
            exr_encoding_choose_default_routines(out, partidx, &encoder);            
        }

        exr_encoding_run(out, partidx, &encoder);
        first = false;
    }
    exr_encoding_destroy(out, &encoder);
    exr_finish(&out);
}

[kdt3rd]

nice one, glad you got it working - just as a side note, you do not need to fill in the channel name in the encoder.channels struct - it will inherit those from the exr_add_channel call (and is intended to allow you to specify the channels "out of order", then with the lexical sorting so far required by the file format, you can determine what the output ordering will be. easy for 4 channels, with lots of AOVs, can get a bit more confusing...

[arabadzhiyski]

thanks!! i was just about to tackle that channel business there to handle additional outputs next... let me see - Do the exr_add_channel() calls have to be in a lexical order? Or will lexical sorting happen regardless of the order in which the channels are added upon calling the function?
I was planning to store the channel names in the same [arbitrary] order as in the planar data layout and do lexicographical sorting on that for the corresponding indices [to use while adding channels and filling-in pointers to channels]. I hope that makes sense.

[arabadzhiyski]

It seems I figured it out:

• exr_add_channel() does not have to be called in lexicographical order of the channel names
• by the time the channels (added by exr_add_channel()) reach the encoding pipeline they will be sorted in lexicographical order

For others who may be looking for examples, here's one [rather hideous looking].

void writeEXR(const Render *picture, char *filename) {
    int pixel_count = picture->width * picture->height;
    int channel_count = 0;
    AOV aov = 0;
    
    // Count the total number of channels for all AOVs
    // We only have a total of 5 supported AOV types to count for
    for (int i = 0; i < picture->naovs; i++ ) {
        aov = picture->outputs[i]->aov;

        if (aov == AOV_BEAUTY || aov == AOV_DIFFUSE || aov == AOV_SPECULAR) channel_count += 3;
        else if (aov == AOV_ALPHA || aov == AOV_SHADOWS)                    channel_count += 1;
        else printf("Invalid AOV in writeEXR()\n");
    }

    //====================================== RAW DATA LAYOUT ======================================
    // Write channel bytes in contiguous manner in an arbitrary AOV order (defined in user settings) 

    size_t pixel_stride = sizeof(float);
    size_t channel_size = pixel_count * pixel_stride; 
    size_t total_size = channel_count * channel_size;

    uint8_t *data = malloc(total_size);
    memset(data, 0, total_size);

    unsigned index = 0;
    int n = 0;

    for (int i = 0; i < picture->naovs; i++) {
        aov = picture->outputs[i]->aov;
        
        if (aov == AOV_BEAUTY || aov == AOV_DIFFUSE || aov == AOV_SPECULAR) {
            for (int y = 0; y < picture->height; y++) { // Iterate over the lines
                for (int x = 0; x < picture->width; x++) { // Iterate over the line pixels
                    index = (y * picture->width + x) * pixel_stride;
                    // This should give us planar layout (e.g. 4x1 px): 
                    // 0__________________   1__________________   2__________________
                    // 0___ 4___ 8___ 12__   0___ 4___ 8___ 12__   0___ 4___ 8___ 12__
                    // RRRR RRRR RRRR RRRR | GGGG GGGG GGGG GGGG | BBBB BBBB BBBB BBBB
                    memcpy(data + (channel_size * (n + 0) + index), &picture->outputs[i]->pixels[y][x].color->r, pixel_stride);
                    memcpy(data + (channel_size * (n + 1) + index), &picture->outputs[i]->pixels[y][x].color->g, pixel_stride);
                    memcpy(data + (channel_size * (n + 2) + index), &picture->outputs[i]->pixels[y][x].color->b, pixel_stride);
                }
            }
            n += 3;
        }
        else if (aov == AOV_ALPHA || aov == AOV_SHADOWS) {
            for (int y = 0; y < picture->height; y++) { // Iterate over the lines
                for (int x = 0; x < picture->width; x++) { // Iterate over the line pixels
                    index = (y * picture->width + x) * pixel_stride;
                    memcpy(data + (channel_size * n + index), picture->outputs[i]->pixels[y][x].alpha, pixel_stride);
                }
            }
            n += 1;
        }
        else {
            printf("Invalid AOV in picture->outputs in writeEXR()\n");
        }
    }

    //======================================== INIT EXR ========================================
    exr_context_initializer_t ctxinit = EXR_DEFAULT_CONTEXT_INITIALIZER;
    ctxinit.max_image_width = picture->width;
    ctxinit.max_image_height = picture->height;

    exr_context_t out;
    int partidx = 0;

    exr_start_write(&out, filename, EXR_WRITE_FILE_DIRECTLY, &ctxinit);

    exr_get_count(out, &partidx);
    assert(partidx == 0);
    exr_add_part(out, "Ci", EXR_STORAGE_SCANLINE, &partidx);
    assert(partidx == 0);

    exr_initialize_required_attr_simple(out, partidx, picture->width, picture->height, EXR_COMPRESSION_RLE);

    //======================================= ADD CHANNELS ===================================
    // Construct the channel names to form virtual channel groups (e.g. diffuse.R, diffuse.G, diffuse.B)
    // Channels R, G, B and A of the 'beauty' AOV will be named 'R', 'G', 'B', 'A' and thus
    // not belong to a group. It appears the [upper] case of these 4 letters may be important.

    char channel_name[128];
    char rgb[3][2] = {"R", "G", "B"};
    char **chlist = malloc(sizeof(char*) * channel_count);
    n = 0;

    for (int i = 0; i < picture->naovs; i++) {
        aov = picture->outputs[i]->aov;

        if (aov == AOV_BEAUTY) {
            for (int j = 0; j < 3; j++) {
                exr_add_channel(out, partidx, rgb[j], EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);

                chlist[n] = malloc(sizeof(char) * 2);
                strcpy(chlist[n], rgb[j]);
                n++;
            }
        }
        else if (aov == AOV_DIFFUSE || aov == AOV_SPECULAR) {
            for (int j = 0; j < 3; j++) {
                sprintf(channel_name, "%s.%s", picture->outputs[i]->name, rgb[j]);

                exr_add_channel(out, partidx, channel_name, EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LOGARITHMIC, 1, 1);

                chlist[n] = malloc(sizeof(char) * strlen(picture->outputs[i]->name) + 3);
                strcpy(chlist[n], channel_name);
                n++;
            }
        }
        else if (aov == AOV_ALPHA || aov == AOV_SHADOWS) {
            exr_add_channel(out, partidx, picture->outputs[i]->name, EXR_PIXEL_FLOAT, EXR_PERCEPTUALLY_LINEAR, 1, 1);

            chlist[n] = malloc(sizeof(char) * strlen(picture->outputs[i]->name) + 1);
            strcpy(chlist[n], picture->outputs[i]->name);
            n++;
        }
        else {
            printf("Invalid AOV in writeEXR()\n");
        }
    }

    // Get a list of indices for the channel names that if applied to the list
    // would sort it in a lexical order. We will use the indices of this argsort
    // to offset the data pointers for the encoding pipeline. Essentially,
    // the index is the channel number in the contiguous data allocation.
    int *chidx = lexisort(chlist, channel_count);
    
    exr_write_header(out);

    //================================ PREPARE FOR ENCODING ===================================
    int nchunks = 0;
    exr_get_chunk_count(out, partidx, &nchunks);

    int lines_per_chink = 0;
    exr_get_scanlines_per_chunk(out, partidx, &lines_per_chink);

    size_t line_stride = pixel_stride * picture->width;
    size_t chunk_stride = line_stride * lines_per_chink;
    size_t step = 0;
    exr_chunk_info_t cinfo;
    exr_encode_pipeline_t encoder;
    bool first = true;
    
    //======================================= ENCODE ===========================================
    for (int y = 0; y < nchunks; y++) {
        exr_write_scanline_chunk_info(out, partidx, y, &cinfo);
        
        if (first) exr_encoding_initialize(out, partidx, &cinfo, &encoder);
        else       exr_encoding_update(out, partidx, &cinfo, &encoder);

        step = chunk_stride * y;

        for (int i = 0; i < encoder.channel_count; i++) {
            // Here's the chidx[i] (i.e. channel number) that times the channel size moves
            // us to the starting location of each channel allocated in 'data'
            encoder.channels[i].encode_from_ptr = &data[channel_size * chidx[i] + step];
            encoder.channels[i].user_data_type = EXR_PIXEL_FLOAT;
            encoder.channels[i].user_pixel_stride = pixel_stride;
            encoder.channels[i].user_line_stride = line_stride;
        }

        if (first) exr_encoding_choose_default_routines(out, partidx, &encoder);

        exr_encoding_run(out, partidx, &encoder);
        first = false;
    }
    
    //======================================= CLEANUP ===========================================
    for (int i = 0; i < channel_count; i++) {
        free(chlist[i]);
    }
    free(chlist);
    free(chidx);
    exr_encoding_destroy(out, &encoder);
    exr_finish(&out);
}