Support rendering YUV formatted frames

data/debayer.frag

@@ -20,49 +20,49 @@ varying vec2 bottom_right_uv;
 vec2
 skip_5th_pixel(vec2 uv)
 {
-        vec2 new_uv = uv;
+    vec2 new_uv = uv;
 
-        new_uv.x *= 0.8;
+    new_uv.x *= 0.8;
-        new_uv.x += floor(uv.x * row_length / 5.0) / row_length;
+    new_uv.x += floor(uv.x * row_length / 5.0) / row_length;
 
-        // Crop out padding
+    // Crop out padding
-        new_uv.x *= padding_ratio;
+    new_uv.x *= padding_ratio;
 
-        return new_uv;
+    return new_uv;
 }
 #endif
 
 void
 main()
 {
-        // Note the coordinates for texture samples need to be a varying, as the
+    // Note the coordinates for texture samples need to be a varying, as the
-        // Mali-400 has this as a fast path allowing 32-bit floats. Otherwise
+    // Mali-400 has this as a fast path allowing 32-bit floats. Otherwise
-        // they end up as 16-bit floats and that's not accurate enough.
+    // they end up as 16-bit floats and that's not accurate enough.
-#ifdef BITS_10
+    #ifdef BITS_10
-        vec4 samples = vec4(texture2D(texture, skip_5th_pixel(top_left_uv)).r,
+    vec4 samples = vec4(texture2D(texture, skip_5th_pixel(top_left_uv)).r,
-                            texture2D(texture, skip_5th_pixel(top_right_uv)).r,
+    texture2D(texture, skip_5th_pixel(top_right_uv)).r,
-                            texture2D(texture, skip_5th_pixel(bottom_left_uv)).r,
+    texture2D(texture, skip_5th_pixel(bottom_left_uv)).r,
-                            texture2D(texture, skip_5th_pixel(bottom_right_uv)).r);
+    texture2D(texture, skip_5th_pixel(bottom_right_uv)).r);
-#else
+    #else
-        vec4 samples = vec4(texture2D(texture, top_left_uv).r,
+    vec4 samples = vec4(texture2D(texture, top_left_uv).r,
-                            texture2D(texture, top_right_uv).r,
+    texture2D(texture, top_right_uv).r,
-                            texture2D(texture, bottom_left_uv).r,
+    texture2D(texture, bottom_left_uv).r,
-                            texture2D(texture, bottom_right_uv).r);
+    texture2D(texture, bottom_right_uv).r);
-#endif
+    #endif
 
-#if defined(CFA_BGGR)
+    #if defined(CFA_BGGR)
-        vec3 color = vec3(samples.w, (samples.y + samples.z) / 2.0, samples.x);
+    vec3 color = vec3(samples.w, (samples.y + samples.z) / 2.0, samples.x);
-#elif defined(CFA_GBRG)
+    #elif defined(CFA_GBRG)
-        vec3 color = vec3(samples.z, (samples.x + samples.w) / 2.0, samples.y);
+    vec3 color = vec3(samples.z, (samples.x + samples.w) / 2.0, samples.y);
-#elif defined(CFA_GRBG)
+    #elif defined(CFA_GRBG)
-        vec3 color = vec3(samples.y, (samples.x + samples.w) / 2.0, samples.z);
+    vec3 color = vec3(samples.y, (samples.x + samples.w) / 2.0, samples.z);
-#else
+    #else
-        vec3 color = vec3(samples.x, (samples.y + samples.z) / 2.0, samples.w);
+    vec3 color = vec3(samples.x, (samples.y + samples.z) / 2.0, samples.w);
-#endif
+    #endif
 
-        color -= blacklevel;
+    color -= blacklevel;
-        color *= color_matrix;
+    color *= color_matrix;
 
-        vec3 gamma_color = pow(color, vec3(inv_gamma));
+    vec3 gamma_color = pow(color, vec3(inv_gamma));
-        gl_FragColor = vec4(gamma_color, 1);
+    gl_FragColor = vec4(gamma_color, 1);
 }

data/debayer.vert

@@ -16,10 +16,10 @@ varying vec2 bottom_right_uv;
 void
 main()
 {
-        top_left_uv = tex_coord - pixel_size / 2.0;
+    top_left_uv = tex_coord - pixel_size / 2.0;
-        bottom_right_uv = tex_coord + pixel_size / 2.0;
+    bottom_right_uv = tex_coord + pixel_size / 2.0;
-        top_right_uv = vec2(top_left_uv.x, bottom_right_uv.y);
+    top_right_uv = vec2(top_left_uv.x, bottom_right_uv.y);
-        bottom_left_uv = vec2(bottom_right_uv.x, top_left_uv.y);
+    bottom_left_uv = vec2(bottom_right_uv.x, top_left_uv.y);
 
-        gl_Position = vec4(transform * vec3(vert, 1), 1);
+    gl_Position = vec4(transform * vec3(vert, 1), 1);
 }

data/org.postmarketos.Megapixels.gresource.xml

@@ -17,5 +17,7 @@
     <file>solid.frag</file>
     <file>debayer.vert</file>
     <file>debayer.frag</file>
+    <file>yuv.vert</file>
+    <file>yuv.frag</file>
   </gresource>
 </gresources>

data/yuv.frag

@@ -0,0 +1,32 @@
+#ifdef GL_ES
+precision highp float;
+#endif
+
+uniform sampler2D texture;
+uniform mat3 color_matrix;
+uniform float inv_gamma;
+uniform float blacklevel;
+
+varying vec2 top_left_uv;
+varying vec2 top_right_uv;
+varying vec2 bottom_left_uv;
+varying vec2 bottom_right_uv;
+
+void
+main()
+{
+    // Note the coordinates for texture samples need to be a varying, as the
+    // Mali-400 has this as a fast path allowing 32-bit floats. Otherwise
+    // they end up as 16-bit floats and that's not accurate enough.
+
+    vec4 samples = vec4(texture2D(texture, top_left_uv).r,
+    texture2D(texture, top_right_uv).r,
+    texture2D(texture, bottom_left_uv).r,
+    texture2D(texture, bottom_right_uv).r);
+    vec3 color = vec3(samples.x, samples.y, samples.z);
+
+    color *= color_matrix;
+    vec3 gamma_color = pow(color, vec3(inv_gamma));
+
+    gl_FragColor = vec4(gamma_color, 1);
+}

data/yuv.vert

@@ -0,0 +1,25 @@
+#ifdef GL_ES
+precision mediump float;
+#endif
+
+attribute vec2 vert;
+attribute vec2 tex_coord;
+
+uniform mat3 transform;
+uniform vec2 pixel_size;
+
+varying vec2 top_left_uv;
+varying vec2 top_right_uv;
+varying vec2 bottom_left_uv;
+varying vec2 bottom_right_uv;
+
+void
+main()
+{
+    top_left_uv = tex_coord - pixel_size / 2.0;
+    bottom_right_uv = tex_coord + pixel_size / 2.0;
+    top_right_uv = vec2(top_left_uv.x, bottom_right_uv.y);
+    bottom_left_uv = vec2(bottom_right_uv.x, top_left_uv.y);
+
+    gl_Position = vec4(transform * vec3(vert, 1), 1);
+}

src/camera.c

@@ -386,6 +386,7 @@ mp_camera_capture_buffer(MPCamera *camera, MPBuffer *buffer)
         }
 
         int format = camera->camera->current_mode->format;
+        assert(format != 0);
         uint32_t width = camera->camera->current_mode->width;
         uint32_t height = camera->camera->current_mode->height;
 

src/gles2_debayer.c

@@ -12,11 +12,21 @@
 GLES2Debayer *
 gles2_debayer_new(int format)
 {
+        // Cannot run on format 0 (Undefined)
+        assert(format != 0);
+
         uint32_t pixfmt = libmegapixels_format_to_v4l_pixfmt(format);
-        if (pixfmt != V4L2_PIX_FMT_SBGGR8 && pixfmt != V4L2_PIX_FMT_SGBRG8 &&
+
-            pixfmt != V4L2_PIX_FMT_SGRBG8 && pixfmt != V4L2_PIX_FMT_SRGGB8 &&
+        int shader = 0;
-            pixfmt != V4L2_PIX_FMT_SBGGR10P && pixfmt != V4L2_PIX_FMT_SGBRG10P &&
+
-            pixfmt != V4L2_PIX_FMT_SGRBG10P && pixfmt != V4L2_PIX_FMT_SRGGB10P) {
+        if (pixfmt == V4L2_PIX_FMT_SBGGR8 || pixfmt == V4L2_PIX_FMT_SGBRG8 ||
+            pixfmt == V4L2_PIX_FMT_SGRBG8 || pixfmt == V4L2_PIX_FMT_SRGGB8 ||
+            pixfmt == V4L2_PIX_FMT_SBGGR10P || pixfmt == V4L2_PIX_FMT_SGBRG10P ||
+            pixfmt == V4L2_PIX_FMT_SGRBG10P || pixfmt == V4L2_PIX_FMT_SRGGB10P) {
+                shader = SHADER_DEBAYER;
+        } else if (pixfmt == V4L2_PIX_FMT_YUYV) {
+                shader = SHADER_YUV;
+        } else {
                 return NULL;
         }
 
@@ -33,17 +43,22 @@ gles2_debayer_new(int format)
 
         const GLchar *def[1] = { format_def };
 
-        GLuint shaders[] = {
+        char shader_vertex[64];
-                gl_util_load_shader("/org/postmarketos/Megapixels/debayer.vert",
+        char shader_fragment[64];
-                                    GL_VERTEX_SHADER,
+        snprintf(shader_vertex,
-                                    NULL,
+                 64,
-                                    0),
+                 "/org/postmarketos/Megapixels/%s.vert",
-                gl_util_load_shader("/org/postmarketos/Megapixels/debayer.frag",
+                 shader == SHADER_DEBAYER ? "debayer" : "yuv");
-                                    GL_FRAGMENT_SHADER,
+        snprintf(shader_fragment,
-                                    def,
+                 64,
-                                    1),
+                 "/org/postmarketos/Megapixels/%s.frag",
-        };
+                 shader == SHADER_DEBAYER ? "debayer" : "yuv");
 
+        GLuint shaders[] = {
+                gl_util_load_shader(shader_vertex, GL_VERTEX_SHADER, NULL, 0),
+                gl_util_load_shader(shader_fragment, GL_FRAGMENT_SHADER, def, 1),
+        };
+        printf("Using shader %s and %s\n", shader_vertex, shader_fragment);
         GLuint program = gl_util_link_program(shaders, 2);
         glBindAttribLocation(program, VERTEX_ATTRIBUTE, "vert");
         glBindAttribLocation(program, TEX_COORD_ATTRIBUTE, "tex_coord");
@@ -51,6 +66,7 @@ gles2_debayer_new(int format)
 
         GLES2Debayer *self = malloc(sizeof(GLES2Debayer));
         self->format = format;
+        self->shader = shader;
 
         self->frame_buffer = frame_buffer;
         self->program = program;
@@ -116,7 +132,9 @@ gles2_debayer_set_shading(GLES2Debayer *self,
                 multiply_matrices(xyzd65, XYZD65_to_sRGB, colormat);
                 glUniformMatrix3fv(
                         self->uniform_color_matrix, 1, GL_FALSE, colormat);
-
+        } else if (self->shader == SHADER_YUV) {
+                glUniformMatrix3fv(
+                        self->uniform_color_matrix, 1, GL_FALSE, YUV_to_RGB);
         } else {
                 glUniformMatrix3fv(
                         self->uniform_color_matrix, 1, GL_FALSE, IDENTITY);

src/gles2_debayer.h

@@ -4,8 +4,12 @@
 #include <assert.h>
 #include <stdio.h>
 
+#define SHADER_DEBAYER 1
+#define SHADER_YUV 2
+
 typedef struct {
         int format;
+        int shader;
         float forward_matrix[9];
 
         GLuint frame_buffer;

src/io_pipeline.c

@@ -457,6 +457,9 @@ update_state(MPPipeline *pipeline, const mp_state_io *new_state)
                         float score = 0;
                         int area_preview =
                                 state_io.preview_width * state_io.preview_height;
+                        if (area_preview == 0) {
+                                area_preview = 1280 * 720;
+                        }
                         for (int m = 0; m < state_io.camera->num_modes; m++) {
                                 float mscore = 0;
                                 if (state_io.camera->modes[m]->rate > 29) {

src/matrix.h

@@ -24,6 +24,14 @@ static float XYZD65_to_sRGB[] = {
         // clang-format on
 };
 
+static float YUV_to_RGB[] = {
+        // clang-format off
+         0.299f,  0.587f,   0.114f,
+        -0.299f, -0.587f,   0.886f,
+         0.701f, -0.597f,  -0.114f
+        // clang-format on
+};
+
 void print_matrix(float m[9]);
 
 void multiply_matrices(const float a[9], const float b[9], float out[9]);