I'll lead with my test case results.
The test case code and implementation is in C#
public void ClosestPointToShouldWork()
{
var r = new Random(0);
double next() => r.NextDouble() * 5 - 1;
var t = new Triangle(new Vector3(0,0,0), new Vector3(3.5,2,0), new Vector3(3,0.0,0));
DrawTriangle(t);
var hash = new Vector3( 0, 0, 0 );
for (int i = 0; i < 800; i++)
{
var pt = new Vector3( next(), next(), 0 );
var pc = t.ClosestPointTo( pt );
hash += pc;
DrawLine(pc,pt);
}
// Test the hash
// If it doesn't match then eyeball the visualization
// and see what has gone wrong
hash.ShouldBeApproximately( new Vector3(1496.28118561104,618.196568578824,0),1e-5 );
}
The implementation code is fiddly as I have a number of framework classes. Hopefully you can treat this as pseudo code and pull out the algorithm. The raw vector types are from https://www.nuget.org/packages/System.DoubleNumerics/.
Note that some of the properties of Triangle could be cached to improve performance.
Note that to return the closest point does not require any square roots and does not require transforming the problem to 2D.
The algorithm first quickly tests if the test point is closest to an end point region. If that is inconclusive it then tests the edge external regions one by one. If those tests fail then the point is inside the triangle. Note that for randomly selected points far from the triangle it is most likely that the closest point will be a corner point of the triangle.
public class Triangle
{
public Vector3 A => EdgeAb.A;
public Vector3 B => EdgeBc.A;
public Vector3 C => EdgeCa.A;
public readonly Edge3 EdgeAb;
public readonly Edge3 EdgeBc;
public readonly Edge3 EdgeCa;
public Triangle(Vector3 a, Vector3 b, Vector3 c)
{
EdgeAb = new Edge3( a, b );
EdgeBc = new Edge3( b, c );
EdgeCa = new Edge3( c, a );
TriNorm = Vector3.Cross(a - b, a - c);
}
public Vector3[] Verticies => new[] {A, B, C};
public readonly Vector3 TriNorm;
private static readonly RangeDouble ZeroToOne = new RangeDouble(0,1);
public Plane TriPlane => new Plane(A, TriNorm);
// The below three could be pre-calculated to
// trade off space vs time
public Plane PlaneAb => new Plane(EdgeAb.A, Vector3.Cross(TriNorm, EdgeAb.Delta ));
public Plane PlaneBc => new Plane(EdgeBc.A, Vector3.Cross(TriNorm, EdgeBc.Delta ));
public Plane PlaneCa => new Plane(EdgeCa.A, Vector3.Cross(TriNorm, EdgeCa.Delta ));
public static readonly RangeDouble Zero1 = new RangeDouble(0,1);
public Vector3 ClosestPointTo(Vector3 p)
{
// Find the projection of the point onto the edge
var uab = EdgeAb.Project( p );
var uca = EdgeCa.Project( p );
if (uca > 1 && uab < 0)
return A;
var ubc = EdgeBc.Project( p );
if (uab > 1 && ubc < 0)
return B;
if (ubc > 1 && uca < 0)
return C;
if (ZeroToOne.Contains( uab ) && !PlaneAb.IsAbove( p ))
return EdgeAb.PointAt( uab );
if (ZeroToOne.Contains( ubc ) && !PlaneBc.IsAbove( p ))
return EdgeBc.PointAt( ubc );
if (ZeroToOne.Contains( uca ) && !PlaneCa.IsAbove( p ))
return EdgeCa.PointAt( uca );
// The closest point is in the triangle so
// project to the plane to find it
return TriPlane.Project( p );
}
}
And the edge structure
public struct Edge3
{
public readonly Vector3 A;
public readonly Vector3 B;
public readonly Vector3 Delta;
public Edge3(Vector3 a, Vector3 b)
{
A = a;
B = b;
Delta = b -a;
}
public Vector3 PointAt(double t) => A + t * Delta;
public double LengthSquared => Delta.LengthSquared();
public double Project(Vector3 p) => (p - A).Dot( Delta ) / LengthSquared;
}
And the plane structure
public struct Plane
{
public Vector3 Point;
public Vector3 Direction;
public Plane(Vector3 point, Vector3 direction )
{
Point = point;
Direction = direction;
}
public bool IsAbove(Vector3 q) => Direction.Dot(q - Point) > 0;
}