avformat/whip: simplify and modularize the ICE and DTLS

We try to perform dtls handshake when the ICE is totally done.

Refer to RFC8445,
When peer's ICE is lite, the peer won't trigged check so FFmpeg just send
STUN request and receive response, then ICE is done.
When peer's ICE is full, the peer will send STUN request after reponse
FFmpeg's request to ensure candidate pair become valid in both directions.
Then the peer does nomination, and ICE is done.

Signed-off-by: Jack Lau <jacklau1222@qq.com>

Author: JackLau1222

libavformat/whip.c

@@ -373,19 +373,6 @@ static av_cold int certificate_key_init(AVFormatContext *s)
     return ret;
 }
 
-static av_cold int dtls_initialize(AVFormatContext *s)
-{
-    WHIPContext *whip = s->priv_data;
-    /* reuse the udp created by whip */
-    ff_tls_set_external_socket(whip->dtls_uc, whip->udp);
-
-    /* Make the socket non-blocking */
-    ff_socket_nonblock(ffurl_get_file_handle(whip->dtls_uc), 1);
-    whip->dtls_uc->flags |= AVIO_FLAG_NONBLOCK;
-
-    return 0;
-}
-
 /**
  * Initialize and check the options for the WebRTC muxer.
  */
@@ -1232,14 +1219,12 @@ static int udp_connect(AVFormatContext *s)
     return ret;
 }
 
-static int ice_dtls_handshake(AVFormatContext *s)
+static int ice_handshake(AVFormatContext *s)
 {
     int ret = 0, size, i;
     int64_t starttime = av_gettime(), now;
     WHIPContext *whip = s->priv_data;
     int is_dtls_active = whip->flags & WHIP_FLAG_DTLS_ACTIVE;
-    AVDictionary *opts = NULL;
-    char buf[256], *cert_buf = NULL, *key_buf = NULL;
 
     if (whip->state < WHIP_STATE_UDP_CONNECTED || !whip->udp) {
         av_log(whip, AV_LOG_ERROR, "UDP not connected, state=%d, udp=%p\n", whip->state, whip->udp);
@@ -1261,25 +1246,20 @@ static int ice_dtls_handshake(AVFormatContext *s)
                 goto end;
             }
 
-            if (whip->state < WHIP_STATE_ICE_CONNECTING)
-                whip->state = WHIP_STATE_ICE_CONNECTING;
+            whip->state = WHIP_STATE_ICE_CONNECTING;
         }
 
 next_packet:
-        if (whip->state >= WHIP_STATE_DTLS_FINISHED)
-            /* DTLS handshake is done, exit the loop. */
-            break;
-
         now = av_gettime();
         if (now - starttime >= whip->handshake_timeout * 1000) {
-            av_log(whip, AV_LOG_ERROR, "DTLS handshake timeout=%dms, cost=%dms, elapsed=%dms, state=%d\n",
+            av_log(whip, AV_LOG_ERROR, "ICE handshake timeout=%dms, cost=%dms, elapsed=%dms, state=%d\n",
                 whip->handshake_timeout, ELAPSED(starttime, now), ELAPSED(whip->whip_starttime, now), whip->state);
             ret = AVERROR(ETIMEDOUT);
             goto end;
         }
 
-        /* Read the STUN or DTLS messages from peer. */
-        for (i = 0; i < ICE_DTLS_READ_INTERVAL / 5 && whip->state < WHIP_STATE_ICE_CONNECTED; i++) {
+        /* Read the STUN or DTLS client hello from peer. */
+        for (i = 0; i < ICE_DTLS_READ_INTERVAL / 5; i++) {
             ret = ffurl_read(whip->udp, whip->buf, sizeof(whip->buf));
             if (ret > 0)
                 break;
@@ -1295,35 +1275,8 @@ static int ice_dtls_handshake(AVFormatContext *s)
 
         /* Handle the ICE binding response. */
         if (ice_is_binding_response(whip->buf, ret)) {
-            if (whip->state < WHIP_STATE_ICE_CONNECTED) {
-                if (whip->is_peer_ice_lite)
-                    whip->state = WHIP_STATE_ICE_CONNECTED;
-                whip->whip_ice_time = av_gettime();
-                av_log(whip, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%dms\n",
-                    whip->state, whip->ice_host, whip->ice_port, whip->whip_resource_url ? whip->whip_resource_url : "",
-                    whip->ice_ufrag_remote, whip->ice_ufrag_local, ret, ELAPSED(whip->whip_starttime, av_gettime()));
-
-                ff_url_join(buf, sizeof(buf), "dtls", NULL, whip->ice_host, whip->ice_port, NULL);
-                av_dict_set_int(&opts, "mtu", whip->pkt_size, 0);
-                if (whip->cert_file) {
-                    av_dict_set(&opts, "cert_file", whip->cert_file, 0);
-                } else
-                    av_dict_set(&opts, "cert_pem", whip->cert_buf, 0);
-
-                if (whip->key_file) {
-                    av_dict_set(&opts, "key_file", whip->key_file, 0);
-                } else
-                    av_dict_set(&opts, "key_pem", whip->key_buf, 0);
-                av_dict_set_int(&opts, "external_sock", 1, 0);
-                av_dict_set_int(&opts, "listen", is_dtls_active ? 0 : 1, 0);
-                /* If got the first binding response, start DTLS handshake. */
-                ret = ffurl_open_whitelist(&whip->dtls_uc, buf, AVIO_FLAG_READ_WRITE, &s->interrupt_callback,
-                    &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
-                av_dict_free(&opts);
-                if (ret < 0)
-                    goto end;
-                dtls_initialize(s);
-            }
+            if (whip->is_peer_ice_lite)
+                whip->state = WHIP_STATE_ICE_CONNECTED;
             goto next_packet;
         }
 
@@ -1334,29 +1287,62 @@ static int ice_dtls_handshake(AVFormatContext *s)
             goto next_packet;
         }
 
-        if ((is_dtls_packet(whip->buf, ret) || is_dtls_active) && whip->state >= WHIP_STATE_ICE_CONNECTED || whip->state == WHIP_STATE_ICE_CONNECTING) {
+        if (is_dtls_packet(whip->buf, ret) || whip->flags & WHIP_FLAG_DTLS_ACTIVE) {
             whip->state = WHIP_STATE_ICE_CONNECTED;
-            ret = ffurl_handshake(whip->dtls_uc);
-            if (ret < 0) {
-                whip->state = WHIP_STATE_FAILED;
-                av_log(whip, AV_LOG_VERBOSE, "DTLS session failed\n");
-                goto end;
-            }
-            if (!ret) {
-                whip->state = WHIP_STATE_DTLS_FINISHED;
-                whip->whip_dtls_time = av_gettime();
-                av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n",
-                    ELAPSED(whip->whip_starttime, whip->whip_dtls_time));
-            }
-            goto next_packet;
+            ret = 0;
+            whip->whip_ice_time = av_gettime();
+            av_log(whip, AV_LOG_VERBOSE, "ICE STUN ok, state=%d, url=udp://%s:%d, location=%s, username=%s:%s, res=%dB, elapsed=%dms\n",
+                whip->state, whip->ice_host, whip->ice_port, whip->whip_resource_url ? whip->whip_resource_url : "",
+                whip->ice_ufrag_remote, whip->ice_ufrag_local, ret, ELAPSED(whip->whip_starttime, av_gettime()));
+            break;
         }
     }
+end:
+    return ret;
+}
 
+static int dtls_handshake(AVFormatContext *s)
+{
+    int ret = 0;
+    WHIPContext *whip = s->priv_data;
+    AVDictionary *opts = NULL;
+    char buf[256];
+
+    ff_url_join(buf, sizeof(buf), "dtls", NULL, whip->ice_host, whip->ice_port, NULL);
+    av_dict_set_int(&opts, "mtu", whip->pkt_size, 0);
+    if (whip->cert_file) {
+        av_dict_set(&opts, "cert_file", whip->cert_file, 0);
+    } else
+        av_dict_set(&opts, "cert_pem", whip->cert_buf, 0);
+
+    if (whip->key_file) {
+        av_dict_set(&opts, "key_file", whip->key_file, 0);
+    } else
+        av_dict_set(&opts, "key_pem", whip->key_buf, 0);
+    av_dict_set_int(&opts, "external_sock", 1, 0);
+    av_dict_set_int(&opts, "listen", whip->flags & WHIP_FLAG_DTLS_ACTIVE ? 0 : 1, 0);
+    /* If got the first binding response, start DTLS handshake. */
+    ret = ffurl_open_whitelist(&whip->dtls_uc, buf, AVIO_FLAG_READ_WRITE, &s->interrupt_callback,
+        &opts, s->protocol_whitelist, s->protocol_blacklist, NULL);
+    av_dict_free(&opts);
+    if (ret < 0)
+        goto end;
+        
+    /* reuse the udp created by whip */
+    ff_tls_set_external_socket(whip->dtls_uc, whip->udp);
+    
+    ret = ffurl_handshake(whip->dtls_uc);
+    if (ret < 0) {
+        whip->state = WHIP_STATE_FAILED;
+        av_log(whip, AV_LOG_VERBOSE, "DTLS session failed\n");
+    }
+    if (!ret) {
+        whip->state = WHIP_STATE_DTLS_FINISHED;
+        whip->whip_dtls_time = av_gettime();
+        av_log(whip, AV_LOG_VERBOSE, "DTLS handshake is done, elapsed=%dms\n",
+            ELAPSED(whip->whip_starttime, whip->whip_dtls_time));
+    }
 end:
-    if (cert_buf)
-        av_free(cert_buf);
-    if (key_buf)
-        av_free(key_buf);
     return ret;
 }
 
@@ -1866,7 +1852,10 @@ static av_cold int whip_init(AVFormatContext *s)
     if ((ret = udp_connect(s)) < 0)
         goto end;
 
-    if ((ret = ice_dtls_handshake(s)) < 0)
+    if ((ret = ice_handshake(s)) < 0)
+        goto end;
+    
+    if ((ret = dtls_handshake(s)) < 0)
         goto end;
 
     if ((ret = setup_srtp(s)) < 0)