Fix wrong coordinate system used for fog

The fogQuake3 shader used the model coordinate system for some
calculations, which is not right since it may be scaled (entities using
nonNormalizedAxes). Do calculations in world space instead.

In Unvanquished I believe this only affects some player models. Models
that use IQM skeleton scaling are not affected.

Author: slipher

src/engine/renderer/Material.cpp

@@ -1091,12 +1091,8 @@ void BindShaderFog( Material* material ) {
 
 	// all fogging distance is based on world Z units
 	vec4_t fogDistanceVector;
-	vec3_t local;
-	VectorSubtract( backEnd.orientation.origin, backEnd.viewParms.orientation.origin, local );
-	fogDistanceVector[0] = -backEnd.orientation.modelViewMatrix[2];
-	fogDistanceVector[1] = -backEnd.orientation.modelViewMatrix[6];
-	fogDistanceVector[2] = -backEnd.orientation.modelViewMatrix[10];
-	fogDistanceVector[3] = DotProduct( local, backEnd.viewParms.orientation.axis[0] );
+	VectorCopy( backEnd.viewParms.orientation.axis[ 0 ], fogDistanceVector );
+	fogDistanceVector[ 3 ] = -DotProduct( fogDistanceVector, backEnd.viewParms.orientation.origin );
 
 	// scale the fog vectors based on the fog's thickness
 	VectorScale( fogDistanceVector, fog->tcScale, fogDistanceVector );
@@ -1106,12 +1102,9 @@ void BindShaderFog( Material* material ) {
 	float eyeT;
 	vec4_t fogDepthVector;
 	if ( fog->hasSurface ) {
-		fogDepthVector[0] = DotProduct( fog->surface, backEnd.orientation.axis[ 0 ] );
-		fogDepthVector[1] = DotProduct( fog->surface, backEnd.orientation.axis[ 1 ] );
-		fogDepthVector[2] = DotProduct( fog->surface, backEnd.orientation.axis[ 2 ] );
-		fogDepthVector[3] = -fog->surface[3] + DotProduct( backEnd.orientation.origin, fog->surface );
-
-		eyeT = DotProduct( backEnd.orientation.viewOrigin, fogDepthVector ) + fogDepthVector[3];
+		VectorCopy( fog->surface, fogDepthVector );
+		fogDepthVector[ 3 ] = -fog->surface[ 3 ];
+		eyeT = DotProduct( backEnd.viewParms.orientation.origin, fogDepthVector ) + fogDepthVector[ 3 ];
 	} else {
 		Vector4Set( fogDepthVector, 0, 0, 0, 1 );
 		eyeT = 1; // non-surface fog always has eye inside

src/engine/renderer/glsl_source/fogQuake3_vp.glsl

@@ -34,12 +34,10 @@ uniform colorPack u_ColorGlobal;
 uniform mat4		u_ModelMatrix;
 uniform mat4		u_ModelViewProjectionMatrix;
 
-uniform vec4		u_FogDistanceVector; // view axis in model coordinates, scaled by fog density
-uniform vec4		u_FogDepthVector; // fog plane in model coordinates
+uniform vec4		u_FogDistanceVector; // view axis, scaled by fog density
+uniform vec4		u_FogDepthVector; // fog plane
 uniform float		u_FogEyeT;
 
-OUT(smooth) vec3	var_Position;
-
 // var_TexCoords.s is distance from viewer to vertex dotted with view axis
 // var_TexCoords.t is the fraction of the viewer-to-vertex ray which is inside fog
 OUT(smooth) vec2	var_TexCoords;
@@ -74,7 +72,8 @@ void main()
 	gl_Position = u_ModelViewProjectionMatrix * position;
 
 	// transform position into world space
-	var_Position = (u_ModelMatrix * position).xyz;
+	position = u_ModelMatrix * position;
+	position.xyz /= position.w;
 
 	// calculate the length in fog
 	float s = dot(position.xyz, u_FogDistanceVector.xyz) + u_FogDistanceVector.w;

src/engine/renderer/tr_shade.cpp

@@ -1597,12 +1597,8 @@ void Render_fog( shaderStage_t* pStage )
 
 	// all fogging distance is based on world Z units
 	vec4_t fogDistanceVector;
-	vec3_t local;
-	VectorSubtract( backEnd.orientation.origin, backEnd.viewParms.orientation.origin, local );
-	fogDistanceVector[ 0 ] = -backEnd.orientation.modelViewMatrix[ 2 ];
-	fogDistanceVector[ 1 ] = -backEnd.orientation.modelViewMatrix[ 6 ];
-	fogDistanceVector[ 2 ] = -backEnd.orientation.modelViewMatrix[ 10 ];
-	fogDistanceVector[ 3 ] = DotProduct( local, backEnd.viewParms.orientation.axis[ 0 ] );
+	VectorCopy( backEnd.viewParms.orientation.axis[ 0 ], fogDistanceVector );
+	fogDistanceVector[ 3 ] = -DotProduct( fogDistanceVector, backEnd.viewParms.orientation.origin );
 
 	// scale the fog vectors based on the fog's thickness
 	VectorScale( fogDistanceVector, fog->tcScale, fogDistanceVector );
@@ -1613,12 +1609,9 @@ void Render_fog( shaderStage_t* pStage )
 	vec4_t fogDepthVector;
 	if ( fog->hasSurface )
 	{
-		fogDepthVector[ 0 ] = DotProduct( fog->surface, backEnd.orientation.axis[ 0 ] );
-		fogDepthVector[ 1 ] = DotProduct( fog->surface, backEnd.orientation.axis[ 1 ] );
-		fogDepthVector[ 2 ] = DotProduct( fog->surface, backEnd.orientation.axis[ 2 ] );
-		fogDepthVector[ 3 ] = -fog->surface[ 3 ] + DotProduct( backEnd.orientation.origin, fog->surface );
-
-		eyeT = DotProduct( backEnd.orientation.viewOrigin, fogDepthVector ) + fogDepthVector[ 3 ];
+		VectorCopy( fog->surface, fogDepthVector );
+		fogDepthVector[ 3 ] = -fog->surface[ 3 ];
+		eyeT = DotProduct( backEnd.viewParms.orientation.origin, fogDepthVector ) + fogDepthVector[ 3 ];
 	}
 	else
 	{