Skip to content

Webcodecs: Handle limited range pixel values in NV12 data. #1606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: webcodecs
Choose a base branch
from
Open

Webcodecs: Handle limited range pixel values in NV12 data. #1606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5e76a5a
Webcodecs: Handle limited range pixel values in
scriby Oct 9, 2025
207f7d9
Webcodecs: Set NCLX color profile information on
scriby Oct 9, 2025
ff82a6d
Fix a typo in "bt2020ncl" to match the value in
scriby Oct 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 110 additions & 37 deletions libheif/plugins/decoder_webcodecs.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,15 @@
#include "libheif/bitstream.h"
#include "libheif/plugins/nalu_utils.h"

#include <algorithm>
#include <assert.h>
#include <cstring>
#include <emscripten/emscripten.h>
#include <emscripten/bind.h>
#include <memory>
#include <queue>
#include <string>
#include <vector>


struct NALUnit {
Expand Down Expand Up @@ -120,6 +124,7 @@ EM_JS(emscripten::EM_VAL, decode_with_browser_hevc, (const char *codec_ptr, uint
}

const format = decoded.format === 'NV12' ? 'NV12' : 'RGBA';
const fullRange = decoded.colorSpace ? decoded.colorSpace.fullRange : false;
const formatOptions = format === 'NV12' ?
{} :
{'format': format, 'colorSpace': 'srgb'};
Expand All @@ -138,6 +143,10 @@ EM_JS(emscripten::EM_VAL, decode_with_browser_hevc, (const char *codec_ptr, uint
'planes': planes,
'codedWidth': decoded.codedWidth,
'codedHeight': decoded.codedHeight,
'fullRange': fullRange,
'primaries': decoded.colorSpace && decoded.colorSpace.primaries ? decoded.colorSpace.primaries : "",
'transfer': decoded.colorSpace && decoded.colorSpace.transfer ? decoded.colorSpace.transfer : "",
'matrix': decoded.colorSpace && decoded.colorSpace.matrix ? decoded.colorSpace.matrix : "",
}));

decoded.close();
Expand Down Expand Up @@ -407,38 +416,82 @@ static struct heif_error webcodecs_push_data(void* decoder_raw, const void* data
return err;
}

static heif_color_primaries get_heif_primaries(const std::string& p) {
if (p == "bt709") return heif_color_primaries_ITU_R_BT_709_5;
if (p == "bt470bg") return heif_color_primaries_ITU_R_BT_470_6_System_B_G;
if (p == "smpte170m") return heif_color_primaries_ITU_R_BT_601_6;
if (p == "bt2020") return heif_color_primaries_ITU_R_BT_2020_2_and_2100_0;
if (p == "smpte431") return heif_color_primaries_SMPTE_RP_431_2;
if (p == "smpte432") return heif_color_primaries_SMPTE_EG_432_1;
return heif_color_primaries_unspecified;
}

static heif_transfer_characteristics get_heif_transfer(const std::string& t) {
if (t == "bt709") return heif_transfer_characteristic_ITU_R_BT_709_5;
if (t == "smpte170m") return heif_transfer_characteristic_ITU_R_BT_601_6;
if (t == "iec61966-2-1" || t == "srgb") return heif_transfer_characteristic_IEC_61966_2_1;
if (t == "linear") return heif_transfer_characteristic_linear;
if (t == "smpte2084" || t == "pq") return heif_transfer_characteristic_ITU_R_BT_2100_0_PQ;
if (t == "hlg") return heif_transfer_characteristic_ITU_R_BT_2100_0_HLG;
return heif_transfer_characteristic_unspecified;
}

static heif_matrix_coefficients get_heif_matrix(const std::string& m) {
if (m == "rgb") return heif_matrix_coefficients_RGB_GBR;
if (m == "bt709") return heif_matrix_coefficients_ITU_R_BT_709_5;
if (m == "bt470bg") return heif_matrix_coefficients_ITU_R_BT_470_6_System_B_G;
if (m == "smpte170m") return heif_matrix_coefficients_ITU_R_BT_601_6;
if (m == "bt2020-ncl") return heif_matrix_coefficients_ITU_R_BT_2020_2_non_constant_luminance;
return heif_matrix_coefficients_unspecified;
}

static struct heif_error convert_webcodecs_result_to_heif_image(const std::unique_ptr<uint8_t[]>& buffer,
int width, int height,
int y_offset, int y_src_stride,
int uv_offset, int uv_src_stride,
struct heif_image** out_img) {
struct heif_image** out_img,
heif_chroma chroma,
bool is_full_range,
const std::string& primaries,
const std::string& transfer,
const std::string& matrix) {
heif_error err;
bool is_mono = chroma == heif_chroma_monochrome;
err = heif_image_create(width,
height,
heif_colorspace_YCbCr,
heif_chroma_420,
is_mono ? heif_colorspace_monochrome : heif_colorspace_YCbCr,
is_mono ? heif_chroma_monochrome : heif_chroma_420,
out_img);
if (err.code) {
return err;
}

struct heif_color_profile_nclx* nclx = heif_nclx_color_profile_alloc();
nclx->color_primaries = get_heif_primaries(primaries);
nclx->transfer_characteristics = get_heif_transfer(transfer);
nclx->matrix_coefficients = get_heif_matrix(matrix);
nclx->full_range_flag = is_full_range ? 1 : 0;
heif_image_set_nclx_color_profile(*out_img, nclx);
heif_nclx_color_profile_free(nclx);

err = heif_image_add_plane(*out_img, heif_channel_Y, width, height, 8);
if (err.code) {
heif_image_release(*out_img);
return err;
}

err = heif_image_add_plane(*out_img, heif_channel_Cb, width / 2, height / 2, 8);
if (err.code) {
heif_image_release(*out_img);
return err;
}
if (!is_mono) {
err = heif_image_add_plane(*out_img, heif_channel_Cb, width / 2, height / 2, 8);
if (err.code) {
heif_image_release(*out_img);
return err;
}

err = heif_image_add_plane(*out_img, heif_channel_Cr, width / 2, height / 2, 8);
if (err.code) {
heif_image_release(*out_img);
return err;
err = heif_image_add_plane(*out_img, heif_channel_Cr, width / 2, height / 2, 8);
if (err.code) {
heif_image_release(*out_img);
return err;
}
}

// The y plane can be reused as-is.
Expand All @@ -452,20 +505,22 @@ static struct heif_error convert_webcodecs_result_to_heif_image(const std::uniqu
width);
}

// In the NV12 format, the U and V planes are interleaved (UVUVUV...), whereas
// in libheif they are two separate planes. This code splits the interleaved UV
// bytes into two separate planes for use in libheif.

int u_stride;
uint8_t* u_dst = heif_image_get_plane(*out_img, heif_channel_Cb, &u_stride);
int v_stride;
uint8_t* v_dst = heif_image_get_plane(*out_img, heif_channel_Cr, &v_stride);

for (int i = 0; i < height / 2; ++i) {
uint8_t* uv_src = buffer.get() + uv_offset + i * uv_src_stride;
for (int j = 0; j < width / 2; ++j) {
u_dst[i * u_stride + j] = uv_src[j * 2];
v_dst[i * v_stride + j] = uv_src[j * 2 + 1];
if (!is_mono) {
// In the NV12 format, the U and V planes are interleaved (UVUVUV...), whereas
// in libheif they are two separate planes. This code splits the interleaved UV
// bytes into two separate planes for use in libheif.

int u_stride;
uint8_t* u_dst = heif_image_get_plane(*out_img, heif_channel_Cb, &u_stride);
int v_stride;
uint8_t* v_dst = heif_image_get_plane(*out_img, heif_channel_Cr, &v_stride);

for (int i = 0; i < height / 2; ++i) {
uint8_t* uv_src = buffer.get() + uv_offset + i * uv_src_stride;
for (int j = 0; j < width / 2; ++j) {
u_dst[i * u_stride + j] = uv_src[j * 2];
v_dst[i * v_stride + j] = uv_src[j * 2 + 1];
}
}
}

Expand Down Expand Up @@ -645,10 +700,15 @@ static struct heif_error webcodecs_decode_image(void* decoder_raw,
// Most HEIC images in the browser will be decoded natively in NV12 pixel
// format. Using the bytes directly helps retain the original image fidelity.
if (format == "NV12") {
if (planes["length"].as<size_t>() < 2) {
bool is_mono = config.chroma_format == 0;
if (!is_mono && planes["length"].as<size_t>() < 2) {
return {heif_error_Decoder_plugin_error,
heif_suberror_Unspecified,
"Decoding failed: NV12 format requires at least 2 planes"};
} else if (is_mono && planes["length"].as<size_t>() < 1) {
return {heif_error_Decoder_plugin_error,
heif_suberror_Unspecified,
"Decoding failed: NV12 monochrome format requires at least 1 plane"};
}

emscripten::val y_plane = planes[0];
Expand All @@ -658,19 +718,32 @@ static struct heif_error webcodecs_decode_image(void* decoder_raw,
"Decoding failed: result.planes[0] is undefined"};
}

emscripten::val uv_plane = planes[1];
if (uv_plane.isUndefined()) {
return {heif_error_Decoder_plugin_error,
heif_suberror_Unspecified,
"Decoding failed: result.planes[1] is undefined"};
}

const int y_offset = y_plane["offset"].as<int>();
const int y_src_stride = y_plane["stride"].as<int>();
const int uv_offset = uv_plane["offset"].as<int>();
const int uv_src_stride = uv_plane["stride"].as<int>();
int uv_offset = 0;
int uv_src_stride = 0;

if (!is_mono) {
emscripten::val uv_plane = planes[1];
if (uv_plane.isUndefined()) {
return {heif_error_Decoder_plugin_error,
heif_suberror_Unspecified,
"Decoding failed: result.planes[1] is undefined"};
}

uv_offset = uv_plane["offset"].as<int>();
uv_src_stride = uv_plane["stride"].as<int>();
}

return convert_webcodecs_result_to_heif_image(buffer, width, height, y_offset, y_src_stride, uv_offset, uv_src_stride, out_img);
bool is_full_range = !result["fullRange"].isUndefined() && result["fullRange"].as<bool>();
std::string primaries = result["primaries"].as<std::string>();
std::string transfer = result["transfer"].as<std::string>();
std::string matrix = result["matrix"].as<std::string>();
return convert_webcodecs_result_to_heif_image(buffer, width, height,
y_offset, y_src_stride,
uv_offset, uv_src_stride, out_img,
(heif_chroma)config.chroma_format, is_full_range,
primaries, transfer, matrix);
} else if (format == "RGBA") {
// Also handle RGBA images as a fallback in cases where the browser returns
// something other than NV12. As of now only RGBA is handled as an
Expand Down
20 changes: 20 additions & 0 deletions libheif/plugins/decoder_webcodecs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,23 @@
/*
* HEIF codec.
* Copyright (c) 2025 Dirk Farin <dirk.farin@gmail.com>
*
* This file is part of libheif.
*
* libheif is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as
* published by the Free Software Foundation, either version 3 of
* the License, or (at your option) any later version.
*
* libheif is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with libheif. If not, see <http://www.gnu.org/licenses/>.
*/

#ifndef THIRD_PARTY_LIBHEIF_LIBHEIF_PLUGINS_DECODER_WEBCODECS_H_
#define THIRD_PARTY_LIBHEIF_LIBHEIF_PLUGINS_DECODER_WEBCODECS_H_

Expand Down