diff --git a/Hazel/Tools/PingBuffer.cs b/Hazel/Tools/PingBuffer.cs
new file mode 100644
index 0000000..40bca7b
--- /dev/null
+++ b/Hazel/Tools/PingBuffer.cs
@@ -0,0 +1,62 @@
+using System;
+
+namespace Hazel.Tools
+{
+    public class PingBuffer
+    {
+        private const ushort InvalidatingFactor = ushort.MaxValue / 2;
+
+        private struct PingInfo
+        {
+            public ushort Id;
+            public DateTime SentAt;
+        }
+
+        private PingInfo[] activePings;
+        private int head; // The location of the next usable activePing
+
+        public PingBuffer(int maxPings)
+        {
+            this.activePings = new PingInfo[maxPings];
+
+            // We don't want the first few packets to match id before we set anything.
+            for (int i = 0; i < this.activePings.Length; ++i)
+            {
+                this.activePings[i].Id = InvalidatingFactor;
+            }
+        }
+
+        public void AddPing(ushort id)
+        {
+            lock (this.activePings)
+            {
+                this.activePings[this.head].Id = id;
+                this.activePings[this.head].SentAt = DateTime.UtcNow;
+                this.head++;
+                if (this.head >= this.activePings.Length)
+                {
+                    this.head = 0;
+                }
+            }
+        }
+
+        public bool TryFindPing(ushort id, out DateTime sentAt)
+        {
+            lock (this.activePings)
+            {
+                for (int i = 0; i < this.activePings.Length; ++i)
+                {
+                    if (this.activePings[i].Id == id)
+                    {
+                        sentAt = this.activePings[i].SentAt;
+                        this.activePings[i].Id += InvalidatingFactor;
+                        return true;
+                    }
+                }
+            }
+
+            sentAt = default;
+            return false;
+        }
+    }
+}
diff --git a/Hazel/Udp/UdpConnection.KeepAlive.cs b/Hazel/Udp/UdpConnection.KeepAlive.cs
index 75b4f1d..eaa1563 100644
--- a/Hazel/Udp/UdpConnection.KeepAlive.cs
+++ b/Hazel/Udp/UdpConnection.KeepAlive.cs
@@ -1,6 +1,6 @@
-using System;
+using Hazel.Tools;
+using System;
 using System.Collections.Concurrent;
-using System.Diagnostics;
 using System.Threading;
 
 
@@ -8,30 +8,6 @@ namespace Hazel.Udp
 {
     partial class UdpConnection
     {
-
-        /// <summary>
-        ///     Class to hold packet data
-        /// </summary>
-        public class PingPacket : IRecyclable
-        {
-            private static readonly ObjectPool<PingPacket> PacketPool = new ObjectPool<PingPacket>(() => new PingPacket());
-
-            public readonly Stopwatch Stopwatch = new Stopwatch();
-
-            internal static PingPacket GetObject()
-            {
-                return PacketPool.GetObject();
-            }
-
-            public void Recycle()
-            {
-                Stopwatch.Stop();
-                PacketPool.PutObject(this);
-            }
-        }
-
-        internal ConcurrentDictionary<ushort, PingPacket> activePingPackets = new ConcurrentDictionary<ushort, PingPacket>();
-
         /// <summary>
         ///     The interval from data being received or transmitted to a keepalive packet being sent in milliseconds.
         /// </summary>
@@ -63,6 +39,10 @@ public int KeepAliveInterval
         public int MissingPingsUntilDisconnect { get; set; } = 6;
         private volatile int pingsSinceAck = 0;
 
+        // TODO: Technically, Min(MissingPingsUntilDisconnect + 1, 16) would be better, but I don't want to mess with it.
+        // The real point is that we're bounding the number of active pings.
+        private PingBuffer activePings = new PingBuffer(16);
+
         /// <summary>
         ///     The timer creating keepalive pulses.
         /// </summary>
@@ -116,18 +96,9 @@ private void SendPing()
             bytes[1] = (byte)(id >> 8);
             bytes[2] = (byte)id;
 
-            PingPacket pkt;
-            if (!this.activePingPackets.TryGetValue(id, out pkt))
-            {
-                pkt = PingPacket.GetObject();
-                if (!this.activePingPackets.TryAdd(id, pkt))
-                {
-                    throw new Exception("This shouldn't be possible");
-                }
-            }
-
-            pkt.Stopwatch.Restart();
-
+            // TODO: This could overwrite a date, perhaps we should track pings that are simply never ack'd?
+            this.activePings.AddPing(id);
+            
             WriteBytesToConnection(bytes, bytes.Length);
 
             Statistics.LogReliableSend(0);
@@ -150,18 +121,7 @@ protected void ResetKeepAliveTimer()
         /// </summary>
         private void DisposeKeepAliveTimer()
         {
-            if (this.keepAliveTimer != null)
-            {
-                this.keepAliveTimer.Dispose();
-            }
-
-            foreach (var kvp in activePingPackets)
-            {
-                if (this.activePingPackets.TryRemove(kvp.Key, out var pkt))
-                {
-                    pkt.Recycle();
-                }
-            }
+            this.keepAliveTimer?.Dispose();
         }
     }
 }
\ No newline at end of file
diff --git a/Hazel/Udp/UdpConnection.Reliable.cs b/Hazel/Udp/UdpConnection.Reliable.cs
index 7494913..96d23d8 100644
--- a/Hazel/Udp/UdpConnection.Reliable.cs
+++ b/Hazel/Udp/UdpConnection.Reliable.cs
@@ -427,11 +427,9 @@ private void AcknowledgeMessageId(ushort id)
                     this._pingMs = this._pingMs * .7f + rt * .3f;
                 }
             }
-            else if (this.activePingPackets.TryRemove(id, out PingPacket pingPkt))
+            else if (this.activePings.TryFindPing(id, out DateTime pingPkt))
             {
-                float rt = pingPkt.Stopwatch.ElapsedMilliseconds;
-
-                pingPkt.Recycle();
+                float rt = (float)(DateTime.UtcNow - pingPkt).TotalMilliseconds;
 
                 lock (PingLock)
                 {