diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs b/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs
index e327d4a2875fd8..72b423f265e78a 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs
@@ -8,6 +8,7 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Text.Unicode;
 
 namespace System
 {
@@ -1377,7 +1378,27 @@ internal static void ThrowOverflowOrFormatException<TChar, TInteger>(ParsingStat
         internal static void ThrowFormatException<TChar>(ReadOnlySpan<TChar> value)
             where TChar : unmanaged, IUtfChar<TChar>
         {
-            throw new FormatException(SR.Format(SR.Format_InvalidStringWithValue, value.ToString()));
+            string errorMessage;
+
+            if (typeof(TChar) == typeof(byte))
+            {
+                // Decode the UTF8 value into a string we can include in the error message. We're here
+                // because we failed to parse, which also means the bytes might not be valid UTF8,
+                // so fallback to a message that doesn't include the value if the bytes are invalid.
+                // It's possible after we check the bytes for validity that they could be concurrently
+                // mutated, but if that's happening, all bets are off, anyway, and it simply impacts
+                // which exception is thrown.
+                ReadOnlySpan<byte> bytes = MemoryMarshal.Cast<TChar, byte>(value);
+                errorMessage = Utf8.IsValid(bytes) ?
+                    SR.Format(SR.Format_InvalidStringWithValue, Encoding.UTF8.GetString(bytes)) :
+                    SR.Format_InvalidString;
+            }
+            else
+            {
+                errorMessage = SR.Format(SR.Format_InvalidStringWithValue, value.ToString());
+            }
+
+            throw new FormatException(errorMessage);
         }
 
         [DoesNotReturn]
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ByteTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ByteTests.cs
index 3593c9b4f470e2..0325507355adae 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ByteTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ByteTests.cs
@@ -405,13 +405,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0u, result);
                 }
 
-                Assert.Throws(exceptionType, () => byte.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => byte.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(byte.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0u, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => byte.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(byte i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DecimalTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DecimalTests.cs
index 3a8c9a344d02cd..cf62bd32354494 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DecimalTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DecimalTests.cs
@@ -1037,13 +1037,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
             if (value != null)
             {
                 ReadOnlySpan<byte> valueUtf8 = Encoding.UTF8.GetBytes(value);
-                Assert.Throws(exceptionType, () => decimal.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => decimal.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(decimal.TryParse(valueUtf8, style, provider, out decimal result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => decimal.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         public static IEnumerable<object[]> Remainder_Valid_TestData()
         {
             decimal NegativeZero = new decimal(0, 0, 0, true, 0);
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DoubleTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DoubleTests.cs
index 3c159c0534e7e1..bf5e2261995ee0 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DoubleTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/DoubleTests.cs
@@ -658,13 +658,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
             if (value != null)
             {
                 ReadOnlySpan<byte> valueUtf8 = Encoding.UTF8.GetBytes(value);
-                Assert.Throws(exceptionType, () => double.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => double.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(double.TryParse(valueUtf8, style, provider, out double result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => double.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Fact]
         public static void PositiveInfinity()
         {
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs
index e5b68dfd0bc0d7..1e7f06602c0ec7 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs
@@ -921,13 +921,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
             if (value != null)
             {
                 ReadOnlySpan<byte> valueUtf8 = Encoding.UTF8.GetBytes(value);
-                Assert.Throws(exceptionType, () => float.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => Half.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(float.TryParse(valueUtf8, style, provider, out float result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => Half.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         public static IEnumerable<object[]> ToString_TestData()
         {
             yield return new object[] { -4570.0f, "G", null, "-4570" };
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int128Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int128Tests.cs
index 26fba082e64a83..8c1910e2fbd8dc 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int128Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int128Tests.cs
@@ -476,13 +476,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0, result);
                 }
 
-                Assert.Throws(exceptionType, () => Int128.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => Int128.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(Int128.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => Int128.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(Int128 i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int16Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int16Tests.cs
index 150b035dccb5fc..60414f96e1a8b6 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int16Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int16Tests.cs
@@ -430,13 +430,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0, result);
                 }
 
-                Assert.Throws(exceptionType, () => short.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => short.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(short.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => short.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(short i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int32Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int32Tests.cs
index 0b5f7c8250efe3..b0e33fc98783eb 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int32Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int32Tests.cs
@@ -864,13 +864,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0, result);
                 }
 
-                Assert.Throws(exceptionType, () => int.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => int.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(int.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => int.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [InlineData("N")]
         [InlineData("F")]
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int64Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int64Tests.cs
index 07b79d94333006..ac902ebdbd7c85 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int64Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Int64Tests.cs
@@ -458,13 +458,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0, result);
                 }
 
-                Assert.Throws(exceptionType, () => long.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => long.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(long.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => long.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(long i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/IntPtrTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/IntPtrTests.cs
index f4a16159467f7e..9b2d96e9797fea 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/IntPtrTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/IntPtrTests.cs
@@ -1021,13 +1021,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(default, result);
                 }
 
-                Assert.Throws(exceptionType, () => int.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => nint.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(nint.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(default, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => nint.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [InlineData("N")]
         [InlineData("F")]
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SByteTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SByteTests.cs
index 0f18bb2e61e02a..7b92f123fd498e 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SByteTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SByteTests.cs
@@ -424,13 +424,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0, result);
                 }
 
-                Assert.Throws(exceptionType, () => sbyte.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => sbyte.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(sbyte.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => sbyte.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(sbyte i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SingleTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SingleTests.cs
index 459bdd843f5a9f..25f8d230c80c03 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SingleTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/SingleTests.cs
@@ -597,13 +597,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
             if (value != null)
             {
                 ReadOnlySpan<byte> valueUtf8 = Encoding.UTF8.GetBytes(value);
-                Assert.Throws(exceptionType, () => float.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => float.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(float.TryParse(valueUtf8, style, provider, out float result));
                 Assert.Equal(0, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => float.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Fact]
         public static void PositiveInfinity()
         {
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt128Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt128Tests.cs
index f17444e678da0b..e3d2c474bcd46b 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt128Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt128Tests.cs
@@ -463,13 +463,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0u, result);
                 }
 
-                Assert.Throws(exceptionType, () => UInt128.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => UInt128.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(UInt128.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0u, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => UInt128.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(UInt128 i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt16Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt16Tests.cs
index 25b4c3b944678e..6ddf7c3bfbaab1 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt16Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt16Tests.cs
@@ -402,13 +402,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0u, result);
                 }
 
-                Assert.Throws(exceptionType, () => ushort.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => ushort.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(ushort.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0u, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => ushort.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(ushort i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt32Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt32Tests.cs
index f912fbd03434a6..c2f80bba01dda7 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt32Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt32Tests.cs
@@ -432,13 +432,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0u, result);
                 }
 
-                Assert.Throws(exceptionType, () => uint.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => uint.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(uint.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0u, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => uint.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(uint i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt64Tests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt64Tests.cs
index e47286dca51b27..de44a6f175884f 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt64Tests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UInt64Tests.cs
@@ -445,13 +445,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(0u, result);
                 }
 
-                Assert.Throws(exceptionType, () => ulong.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => ulong.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(ulong.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(0u, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => ulong.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(ulong i, string format, IFormatProvider provider, string expected) =>
diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UIntPtrTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UIntPtrTests.cs
index 6e1f5d3b460945..6978b9e44eb590 100644
--- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UIntPtrTests.cs
+++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/UIntPtrTests.cs
@@ -584,13 +584,26 @@ public static void Parse_Utf8Span_Invalid(string value, NumberStyles style, IFor
                     Assert.Equal(default, result);
                 }
 
-                Assert.Throws(exceptionType, () => nuint.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                Exception e = Assert.Throws(exceptionType, () => nuint.Parse(Encoding.UTF8.GetBytes(value), style, provider));
+                if (e is FormatException fe)
+                {
+                    Assert.Contains(value, fe.Message);
+                }
 
                 Assert.False(nuint.TryParse(valueUtf8, style, provider, out result));
                 Assert.Equal(default, result);
             }
         }
 
+        [Fact]
+        public static void Parse_Utf8Span_InvalidUtf8()
+        {
+            FormatException fe = Assert.Throws<FormatException>(() => nuint.Parse([0xA0]));
+            Assert.DoesNotContain("A0", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("ReadOnlySpan", fe.Message, StringComparison.Ordinal);
+            Assert.DoesNotContain("\uFFFD", fe.Message, StringComparison.Ordinal);
+        }
+
         [Theory]
         [MemberData(nameof(ToString_TestData))]
         public static void TryFormat(nuint i, string format, IFormatProvider provider, string expected) =>
