fix pixel scale

gopro_8:7/unfish_gopro_8:7.glsl

@@ -5,34 +5,61 @@
 
 #extension GL_ARB_texture_rectangle: enable
 
+// TODO: investigate
+// precision highp float;
+
+// uniforms
 uniform sampler2DRect myTextureY;
 uniform sampler2DRect myTextureU;
 uniform sampler2DRect myTextureV;
 uniform vec2 myResolution;
 uniform float pts;
 
-const vec2 half_pixel = vec2(0.5, 0.5);
-
-const vec2 input_scale = vec2(0.652485, 1.0);
+// parameters
 const float input_fov = 156.0;
-const float output_fov = 119.789529;
+const float output_fov = 124.45;
+const vec2 pixel_scale = vec2(0.652485, 1.0);
+const int subsampling = 4;
 
-vec2 unfish(vec2 coord) {
-    float diameter = sqrt(dot(myResolution, myResolution));
-    vec2 center = myResolution * 0.5;
-    vec2 input_position = (coord - center) * input_scale;
-    float input_distance = length(input_position);
-    float input_len = diameter / radians(input_fov);
-    float output_len = diameter / (2.0 * tan(radians(output_fov) * 0.5));
-    float unfish_ratio = input_len * atan(input_distance / output_len) / input_distance;
-    return center + input_position * unfish_ratio;
+// subsampling constants
+const float substep = 1.0 / float(subsampling);
+const float substart = substep * 0.5 - 0.5;
+const float subscale = 1.0 / float(subsampling * subsampling);
+
+// variables
+vec2 center;
+float diameter;
+float input_len;
+float inv_output_len;
+
+vec4 unfish(const in vec2 coord) {
+    float len = max(0.001, length(coord));
+    vec2 y_coord = center + coord * ((input_len / len) * atan(len * inv_output_len));
+    vec2 uv_coord = y_coord * 0.5;
+    return vec4(
+        texture2DRect(myTextureY, y_coord).r,
+        texture2DRect(myTextureU, uv_coord).r,
+        texture2DRect(myTextureV, uv_coord).r,
+        1.0
+    );
 }
 
 void main() {
-    vec2 y_coord = unfish(gl_TexCoord[0].xy + half_pixel);
-    vec2 uv_coord = y_coord * 0.5;
-    vec4 y = texture2DRect(myTextureY, y_coord);
-    vec4 u = texture2DRect(myTextureU, uv_coord);
-    vec4 v = texture2DRect(myTextureV, uv_coord);
-    gl_FragColor = vec4(y.r, u.r, v.r, 1.0);
+    center = myResolution * 0.5;
+    diameter = length(myResolution);
+    input_len = diameter / radians(input_fov);
+    inv_output_len = (2.0 * tan(radians(output_fov * 0.5))) / diameter;
+
+    vec2 coord = gl_TexCoord[0].xy - center;
+    vec4 pixel = vec4(0.0, 0.0, 0.0, 0.0);
+
+    float x, y = substart;
+    for (int column = 0; column < subsampling; column++, y += substep) {
+        x = substart;
+        for (int row = 0; row < subsampling; row++, x += substep) {
+            pixel += unfish((coord + vec2(x, y)) * pixel_scale);
+        }
+    }
+
+    gl_FragColor = pixel * subscale;
 }