From 6b1b4336aeb92b0476062f4a51a644169fe3d03c Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Wed, 22 Aug 2018 23:50:58 +0200
Subject: [PATCH 01/50] Automate _cluster_setup

---
 mem3_helper.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 58 insertions(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 961e969..309fc52 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -50,11 +50,68 @@ def connect_the_dots(names):
             resp = requests.put(uri, data=json.dumps(doc))
         print('Adding cluster member', name, resp.status_code)
 
+# Compare (json) objects - order does not matter. Credits to:
+# https://stackoverflow.com/a/25851972
+def ordered(obj):
+    if isinstance(obj, dict):
+        return sorted((k, ordered(v)) for k, v in obj.items())
+    if isinstance(obj, list):
+        return sorted(ordered(x) for x in obj)
+    else:
+        return obj
+
+def finish_cluster(names):
+    # The HTTP POST to /_cluster_setup should be done to
+    # one (and only one) of the CouchDB cluster nodes.
+    # Given that K8S has a standardized naming for the pods:
+    # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
+    # we can make sure that this code is only bing run
+    # on the "first" pod using this hack:
+    if (os.getenv("HOSTNAME").endswith("_1")):
+        # Make sure that ALL CouchDB cluster peers have been
+        # primed with _nodes data before /_cluster_setup
+        creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
+        # Use the _members of "this" pod's CouchDB as reference
+        if creds[0] and creds[1]:
+            local_resp = requests.get("http://{0}127.0.0.1:5984/_members",  auth=creds)
+        else:
+            local_resp = requests.get("http://{0}127.0.0.1:5984/_members")
+        for name in names:
+            print('Probing ', name)
+            if creds[0] and creds[1]:
+                remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
+            else:
+                remote_resp = requests.get("http://{0}:5984/_members".format(name))
+            while (remote_resp.status_code == 404) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
+                print('Waiting for node {0} to be ready'.format(name))
+                time.sleep(5)
+                if creds[0] and creds[1]:
+                    remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
+                else:
+                    remote_resp = requests.get("http://{0}:5984/_members".format(name))
+            print('CouchDB cluster member {} ready to form a cluster'.format(name))
+        # At this point ALL peers have _nodes populated. Finish the cluster setup!
+        payload = {}
+        if creds[0] and creds[1]:
+            setup_resp=requests.post('http://127.0.0.1:5984/_cluster_setup', json={"action": "finish_cluster"},  auth=creds)
+        else:
+            setup_resp=requests.post('http://127.0.0.1:5984/_cluster_setup', json={"action": "finish_cluster"},  auth=creds)
+        if (setup_resp.status_code == 200)
+            print('CouchDB cluster done. Time to relax!')
+        else:
+            print('Ouch! Failed with final step. http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
+            
+
+
+
+
 def sleep_forever():
     while True:
         time.sleep(5)
 
 if __name__ == '__main__':
-    connect_the_dots(discover_peers(construct_service_record()))
+    peer_names = discover_peers(construct_service_record())
+    connect_the_dots(peer_names)
+    finish_cluster(peer_names)
     print('Cluster membership populated!')
     sleep_forever()

From f20b962a95741be8509bb0fc392a928f191a1988 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 06:48:01 +0200
Subject: [PATCH 02/50] Automate _cluster_setup

---
 mem3_helper.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 309fc52..f3d0e0c 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -67,7 +67,7 @@ def finish_cluster(names):
     # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
     # we can make sure that this code is only bing run
     # on the "first" pod using this hack:
-    if (os.getenv("HOSTNAME").endswith("_1")):
+    if (os.getenv("HOSTNAME").endswith("_0")):
         # Make sure that ALL CouchDB cluster peers have been
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
@@ -77,7 +77,7 @@ def finish_cluster(names):
         else:
             local_resp = requests.get("http://{0}127.0.0.1:5984/_members")
         for name in names:
-            print('Probing ', name)
+            print("Probing. Checking {0}".format(name))
             if creds[0] and creds[1]:
                 remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
             else:
@@ -89,18 +89,19 @@ def finish_cluster(names):
                     remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
                 else:
                     remote_resp = requests.get("http://{0}:5984/_members".format(name))
-            print('CouchDB cluster member {} ready to form a cluster'.format(name))
+            print("CouchDB cluster member {} ready to form a cluster".format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
         payload = {}
         if creds[0] and creds[1]:
-            setup_resp=requests.post('http://127.0.0.1:5984/_cluster_setup', json={"action": "finish_cluster"},  auth=creds)
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         else:
-            setup_resp=requests.post('http://127.0.0.1:5984/_cluster_setup', json={"action": "finish_cluster"},  auth=creds)
-        if (setup_resp.status_code == 200)
-            print('CouchDB cluster done. Time to relax!')
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"})
+        if setup_resp.status_code == 200:
+            print("CouchDB cluster done. Time to relax!")
         else:
             print('Ouch! Failed with final step. http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
-            
+    else:
+        print("This pod is not used to call /_setup_cluster")
 
 
 

From 39a868b619321eaec166a4b9074ace0feedfd9ce Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 07:07:50 +0200
Subject: [PATCH 03/50] Automate _cluster_setup

---
 mem3_helper.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index f3d0e0c..387f35a 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -91,12 +91,11 @@ def finish_cluster(names):
                     remote_resp = requests.get("http://{0}:5984/_members".format(name))
             print("CouchDB cluster member {} ready to form a cluster".format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
-        payload = {}
         if creds[0] and creds[1]:
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         else:
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"})
-        if setup_resp.status_code == 200:
+        if (setup_resp.status_code == 200):
             print("CouchDB cluster done. Time to relax!")
         else:
             print('Ouch! Failed with final step. http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))

From deb25d13a8fd542f35661d2553ecde44b4e098be Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 07:36:44 +0200
Subject: [PATCH 04/50] Automate _cluster_setup

---
 mem3_helper.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 387f35a..9fbced9 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -67,7 +67,7 @@ def finish_cluster(names):
     # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
     # we can make sure that this code is only bing run
     # on the "first" pod using this hack:
-    if (os.getenv("HOSTNAME").endswith("_0")):
+    if True or (os.getenv("HOSTNAME").endswith("_0")):
         # Make sure that ALL CouchDB cluster peers have been
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
@@ -95,12 +95,12 @@ def finish_cluster(names):
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         else:
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"})
-        if (setup_resp.status_code == 200):
+        if (setup_resp.status_code == requests.codes.ok):
             print("CouchDB cluster done. Time to relax!")
         else:
-            print('Ouch! Failed with final step. http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
+            print('Ouch! Failed the final step: http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
     else:
-        print("This pod is not used to call /_setup_cluster")
+        print("This pod is intentionally skipping the call to http://127.0.0.1:5984/_cluster_setup")
 
 
 
@@ -112,6 +112,6 @@ def sleep_forever():
 if __name__ == '__main__':
     peer_names = discover_peers(construct_service_record())
     connect_the_dots(peer_names)
-    finish_cluster(peer_names)
     print('Cluster membership populated!')
+    finish_cluster(peer_names)
     sleep_forever()

From d645ae680c96f0bfff9bde01d51eead2e830b812 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 07:59:17 +0200
Subject: [PATCH 05/50] Automate _cluster_setup

---
 mem3_helper.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 9fbced9..b3cb09f 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -67,7 +67,7 @@ def finish_cluster(names):
     # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
     # we can make sure that this code is only bing run
     # on the "first" pod using this hack:
-    if True or (os.getenv("HOSTNAME").endswith("_0")):
+    if (os.getenv("HOSTNAME").endswith("_0")):
         # Make sure that ALL CouchDB cluster peers have been
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
@@ -77,34 +77,32 @@ def finish_cluster(names):
         else:
             local_resp = requests.get("http://{0}127.0.0.1:5984/_members")
         for name in names:
-            print("Probing. Checking {0}".format(name))
+            print("Probing node {0} for _members".format(name))
             if creds[0] and creds[1]:
                 remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
             else:
                 remote_resp = requests.get("http://{0}:5984/_members".format(name))
             while (remote_resp.status_code == 404) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
-                print('Waiting for node {0} to be ready'.format(name))
+                print('Waiting for node {0} to have all node members populated'.format(name))
                 time.sleep(5)
                 if creds[0] and creds[1]:
                     remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
                 else:
                     remote_resp = requests.get("http://{0}:5984/_members".format(name))
-            print("CouchDB cluster member {} ready to form a cluster".format(name))
+            print("CouchDB cluster peer {} ready to form a cluster".format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
         if creds[0] and creds[1]:
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         else:
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"})
         if (setup_resp.status_code == requests.codes.ok):
-            print("CouchDB cluster done. Time to relax!")
+            print("CouchDB cluster setup done. Time to relax!")
         else:
             print('Ouch! Failed the final step: http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
     else:
         print("This pod is intentionally skipping the call to http://127.0.0.1:5984/_cluster_setup")
 
 
-
-
 def sleep_forever():
     while True:
         time.sleep(5)

From 2365de6cb5c4af56903967381c687a7b4db6e1cf Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 08:23:03 +0200
Subject: [PATCH 06/50] Automate _cluster_setup

---
 mem3_helper.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index b3cb09f..6961ea1 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -72,10 +72,12 @@ def finish_cluster(names):
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         # Use the _members of "this" pod's CouchDB as reference
+        local_members_uri = "http://127.0.0.1:5984/_members"
+        print ("Fetching node mebership from this pod: {0}".format(local_members_uri))
         if creds[0] and creds[1]:
-            local_resp = requests.get("http://{0}127.0.0.1:5984/_members",  auth=creds)
+            local_resp = requests.get(local_members_uri,  auth=creds)
         else:
-            local_resp = requests.get("http://{0}127.0.0.1:5984/_members")
+            local_resp = requests.get(local_members_uri)
         for name in names:
             print("Probing node {0} for _members".format(name))
             if creds[0] and creds[1]:

From a00d15fedefef87b18c92727bf93764ebd110d81 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 08:49:29 +0200
Subject: [PATCH 07/50] Automate _cluster_setup

---
 mem3_helper.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 6961ea1..79c061c 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -23,7 +23,7 @@ def construct_service_record():
     max_tries=10
 )
 def discover_peers(service_record):
-    print ('Resolving SRV record', service_record)
+    print('Resolving SRV record', service_record, flush=True)
     answers = dns.resolver.query(service_record, 'SRV')
     # Erlang requires that we drop the trailing period from the absolute DNS
     # name to form the hostname used for the Erlang node. This feels hacky
@@ -45,10 +45,10 @@ def connect_the_dots(names):
         else:
             resp = requests.put(uri, data=json.dumps(doc))
         while resp.status_code == 404:
-            print('Waiting for _nodes DB to be created ...')
+            print('Waiting for _nodes DB to be created ...', flush=True)
             time.sleep(5)
             resp = requests.put(uri, data=json.dumps(doc))
-        print('Adding cluster member', name, resp.status_code)
+        print('Adding cluster member', name, resp.status_code, flush=True)
 
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972
@@ -72,8 +72,8 @@ def finish_cluster(names):
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         # Use the _members of "this" pod's CouchDB as reference
-        local_members_uri = "http://127.0.0.1:5984/_members"
-        print ("Fetching node mebership from this pod: {0}".format(local_members_uri))
+        local_members_uri = "http://127.0.0.1:5986/_membership"
+        print ("Fetching node mebership from this pod: {0}".format(local_members_uri),flush=True)
         if creds[0] and creds[1]:
             local_resp = requests.get(local_members_uri,  auth=creds)
         else:
@@ -81,16 +81,16 @@ def finish_cluster(names):
         for name in names:
             print("Probing node {0} for _members".format(name))
             if creds[0] and creds[1]:
-                remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
+                remote_resp = requests.get("http://{0}:5984/_membership".format(name),  auth=creds)
             else:
-                remote_resp = requests.get("http://{0}:5984/_members".format(name))
+                remote_resp = requests.get("http://{0}:5984/_membership".format(name))
             while (remote_resp.status_code == 404) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
-                print('Waiting for node {0} to have all node members populated'.format(name))
+                print('Waiting for node {0} to have all node members populated'.format(name),flush=True)
                 time.sleep(5)
                 if creds[0] and creds[1]:
-                    remote_resp = requests.get("http://{0}:5984/_members".format(name),  auth=creds)
+                    remote_resp = requests.get("http://{0}:5984/_membership".format(name),  auth=creds)
                 else:
-                    remote_resp = requests.get("http://{0}:5984/_members".format(name))
+                    remote_resp = requests.get("http://{0}:5984/_membership".format(name))
             print("CouchDB cluster peer {} ready to form a cluster".format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
         if creds[0] and creds[1]:

From bd16519b8750198c45ae06acd67feb70e75ea5b0 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 09:03:50 +0200
Subject: [PATCH 08/50] Automate _cluster_setup

---
 mem3_helper.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 79c061c..4e209b6 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -47,7 +47,10 @@ def connect_the_dots(names):
         while resp.status_code == 404:
             print('Waiting for _nodes DB to be created ...', flush=True)
             time.sleep(5)
-            resp = requests.put(uri, data=json.dumps(doc))
+            if creds[0] and creds[1]:
+                resp = requests.put(uri, data=json.dumps(doc), auth=creds)
+            else:
+                resp = requests.put(uri, data=json.dumps(doc))
         print('Adding cluster member', name, resp.status_code, flush=True)
 
 # Compare (json) objects - order does not matter. Credits to:

From 7719d57e777000fece86eeb0cc9bbb6700fa122d Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 09:51:46 +0200
Subject: [PATCH 09/50] Automate _cluster_setup

---
 mem3_helper.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 4e209b6..9332f50 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -44,7 +44,7 @@ def connect_the_dots(names):
             resp = requests.put(uri, data=json.dumps(doc), auth=creds)
         else:
             resp = requests.put(uri, data=json.dumps(doc))
-        while resp.status_code == 404:
+        while resp.status_code != 201:
             print('Waiting for _nodes DB to be created ...', flush=True)
             time.sleep(5)
             if creds[0] and creds[1]:
@@ -70,7 +70,8 @@ def finish_cluster(names):
     # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
     # we can make sure that this code is only bing run
     # on the "first" pod using this hack:
-    if (os.getenv("HOSTNAME").endswith("_0")):
+    print('HOSTNAME={0}'.format(os.getenv("HOSTNAME")))
+    if (os.getenv("HOSTNAME").endswith("-0")):
         # Make sure that ALL CouchDB cluster peers have been
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
@@ -87,7 +88,7 @@ def finish_cluster(names):
                 remote_resp = requests.get("http://{0}:5984/_membership".format(name),  auth=creds)
             else:
                 remote_resp = requests.get("http://{0}:5984/_membership".format(name))
-            while (remote_resp.status_code == 404) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
+            while (remote_resp.status_code != 200) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
                 print('Waiting for node {0} to have all node members populated'.format(name),flush=True)
                 time.sleep(5)
                 if creds[0] and creds[1]:
@@ -100,7 +101,7 @@ def finish_cluster(names):
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         else:
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"})
-        if (setup_resp.status_code == requests.codes.ok):
+        if (setup_resp.status_code == 201):
             print("CouchDB cluster setup done. Time to relax!")
         else:
             print('Ouch! Failed the final step: http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))

From ccf6bcf0dbdb721e780b79e0627fb9e07d90d207 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 11:07:17 +0200
Subject: [PATCH 10/50] Automate _cluster_setup

---
 mem3_helper.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 9332f50..5b2fe2b 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -76,25 +76,29 @@ def finish_cluster(names):
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         # Use the _members of "this" pod's CouchDB as reference
-        local_members_uri = "http://127.0.0.1:5986/_membership"
-        print ("Fetching node mebership from this pod: {0}".format(local_members_uri),flush=True)
+        local_membership_uri = "http://127.0.0.1:5986/_membership"
+        print ("Fetching node mebership from this pod: {0}".format(local_membership_uri),flush=True)
         if creds[0] and creds[1]:
-            local_resp = requests.get(local_members_uri,  auth=creds)
+            local_resp = requests.get(local_membership_uri,  auth=creds)
         else:
-            local_resp = requests.get(local_members_uri)
+            local_resp = requests.get(local_membership_uri)
         for name in names:
             print("Probing node {0} for _members".format(name))
+            remote_membership_uri = "http://{0}:5984/_membership".format(name)
             if creds[0] and creds[1]:
-                remote_resp = requests.get("http://{0}:5984/_membership".format(name),  auth=creds)
+                remote_resp = requests.get(remote_membership_uri,  auth=creds)
             else:
-                remote_resp = requests.get("http://{0}:5984/_membership".format(name))
+                remote_resp = requests.get(remote_membership_uri)
             while (remote_resp.status_code != 200) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
+                print ("remote_resp.status_code",remote_resp.status_code)
+                print (ordered(local_resp.json()))
+                print (ordered(remote_resp.json()))
                 print('Waiting for node {0} to have all node members populated'.format(name),flush=True)
                 time.sleep(5)
                 if creds[0] and creds[1]:
-                    remote_resp = requests.get("http://{0}:5984/_membership".format(name),  auth=creds)
+                    remote_resp = requests.get(remote_membership_uri,  auth=creds)
                 else:
-                    remote_resp = requests.get("http://{0}:5984/_membership".format(name))
+                    remote_resp = requests.get(remote_membership_uri)
             print("CouchDB cluster peer {} ready to form a cluster".format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
         if creds[0] and creds[1]:

From 7171c90b3e73652a8cb77b9654c2557eab459874 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 11:30:15 +0200
Subject: [PATCH 11/50] Automate _cluster_setup

---
 mem3_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 5b2fe2b..3eb691c 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -76,7 +76,7 @@ def finish_cluster(names):
         # primed with _nodes data before /_cluster_setup
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         # Use the _members of "this" pod's CouchDB as reference
-        local_membership_uri = "http://127.0.0.1:5986/_membership"
+        local_membership_uri = "http://127.0.0.1:5984/_membership"
         print ("Fetching node mebership from this pod: {0}".format(local_membership_uri),flush=True)
         if creds[0] and creds[1]:
             local_resp = requests.get(local_membership_uri,  auth=creds)

From 1bc4d65229fe539b916b73008b3140694b0c75c1 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 12:32:27 +0200
Subject: [PATCH 12/50] Automate _cluster_setup

---
 mem3_helper.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mem3_helper.py b/mem3_helper.py
index 3eb691c..9e9240f 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -119,6 +119,7 @@ def sleep_forever():
 
 if __name__ == '__main__':
     peer_names = discover_peers(construct_service_record())
+    print('Got the following peers fqdm from DNS lookup:',peer_names,flush=True)
     connect_the_dots(peer_names)
     print('Cluster membership populated!')
     finish_cluster(peer_names)

From 7f93d0653b0934bdb09def531b1681c96d530db7 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 13:42:57 +0200
Subject: [PATCH 13/50] Automate _cluster_setup

---
 mem3_helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 9e9240f..8d015a7 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -45,7 +45,7 @@ def connect_the_dots(names):
         else:
             resp = requests.put(uri, data=json.dumps(doc))
         while resp.status_code != 201:
-            print('Waiting for _nodes DB to be created ...', flush=True)
+            print('Waiting for _nodes DB to be created.',uri,'returned', resp.status_code, flush=True)
             time.sleep(5)
             if creds[0] and creds[1]:
                 resp = requests.put(uri, data=json.dumps(doc), auth=creds)
@@ -119,7 +119,7 @@ def sleep_forever():
 
 if __name__ == '__main__':
     peer_names = discover_peers(construct_service_record())
-    print('Got the following peers fqdm from DNS lookup:',peer_names,flush=True)
+    print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
     connect_the_dots(peer_names)
     print('Cluster membership populated!')
     finish_cluster(peer_names)

From 970510bf92099c9fc4f465672f5c6e2860a067e0 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 14:17:26 +0200
Subject: [PATCH 14/50] Automate _cluster_setup

---
 mem3_helper.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 8d015a7..8960a4b 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -51,7 +51,7 @@ def connect_the_dots(names):
                 resp = requests.put(uri, data=json.dumps(doc), auth=creds)
             else:
                 resp = requests.put(uri, data=json.dumps(doc))
-        print('Adding cluster member', name, resp.status_code, flush=True)
+        print('Adding CouchDB cluster node', name, "to this pod's CouchDB", flush=True)
 
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972
@@ -72,27 +72,32 @@ def finish_cluster(names):
     # on the "first" pod using this hack:
     print('HOSTNAME={0}'.format(os.getenv("HOSTNAME")))
     if (os.getenv("HOSTNAME").endswith("-0")):
+        creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         # Make sure that ALL CouchDB cluster peers have been
         # primed with _nodes data before /_cluster_setup
-        creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
-        # Use the _members of "this" pod's CouchDB as reference
+        # Use the _membership of "this" pod's CouchDB as reference
         local_membership_uri = "http://127.0.0.1:5984/_membership"
-        print ("Fetching node mebership from this pod: {0}".format(local_membership_uri),flush=True)
+        print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
         if creds[0] and creds[1]:
             local_resp = requests.get(local_membership_uri,  auth=creds)
         else:
             local_resp = requests.get(local_membership_uri)
+        # Step through every peer pod and grab the _membership.
         for name in names:
-            print("Probing node {0} for _members".format(name))
+            print("Probing {0} for cluster membership".format(name))
             remote_membership_uri = "http://{0}:5984/_membership".format(name)
             if creds[0] and creds[1]:
                 remote_resp = requests.get(remote_membership_uri,  auth=creds)
             else:
                 remote_resp = requests.get(remote_membership_uri)
+            # Compare local and remote _mebership data. Make sure the set
+            # of nodes match. This will ensure that the remote nodes
+            # are fully primed with nodes data before progressing with
+            # _cluster_setup
             while (remote_resp.status_code != 200) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
-                print ("remote_resp.status_code",remote_resp.status_code)
-                print (ordered(local_resp.json()))
-                print (ordered(remote_resp.json()))
+                # print ("remote_resp.status_code",remote_resp.status_code)
+                # print (ordered(local_resp.json()))
+                # print (ordered(remote_resp.json()))
                 print('Waiting for node {0} to have all node members populated'.format(name),flush=True)
                 time.sleep(5)
                 if creds[0] and creds[1]:

From 9367e6b609de5c0e0afbaca230b4aab75000a8fa Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 15:17:25 +0200
Subject: [PATCH 15/50] Automate _cluster_setup

---
 mem3_helper.py | 41 +++++++++++++++++++++++++++++++----------
 1 file changed, 31 insertions(+), 10 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 8960a4b..e597d4b 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -63,13 +63,16 @@ def ordered(obj):
     else:
         return obj
 
+# Run action:finish_cluster on one and only one CouchDB cluster node
 def finish_cluster(names):
     # The HTTP POST to /_cluster_setup should be done to
     # one (and only one) of the CouchDB cluster nodes.
+    # Search for "setup coordination node" in
+    # http://docs.couchdb.org/en/stable/cluster/setup.html
     # Given that K8S has a standardized naming for the pods:
     # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
-    # we can make sure that this code is only bing run
-    # on the "first" pod using this hack:
+    # we can make sure that this code runs
+    # on the "first" pod only with this hack:
     print('HOSTNAME={0}'.format(os.getenv("HOSTNAME")))
     if (os.getenv("HOSTNAME").endswith("-0")):
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
@@ -88,8 +91,6 @@ def finish_cluster(names):
             remote_membership_uri = "http://{0}:5984/_membership".format(name)
             if creds[0] and creds[1]:
                 remote_resp = requests.get(remote_membership_uri,  auth=creds)
-            else:
-                remote_resp = requests.get(remote_membership_uri)
             # Compare local and remote _mebership data. Make sure the set
             # of nodes match. This will ensure that the remote nodes
             # are fully primed with nodes data before progressing with
@@ -102,14 +103,23 @@ def finish_cluster(names):
                 time.sleep(5)
                 if creds[0] and creds[1]:
                     remote_resp = requests.get(remote_membership_uri,  auth=creds)
-                else:
-                    remote_resp = requests.get(remote_membership_uri)
-            print("CouchDB cluster peer {} ready to form a cluster".format(name))
+            # The node in <name> is primed
+            # http://docs.couchdb.org/en/stable/cluster/setup.html
+
+            # action: enable_cluster
+            payload = '"action": "enable_cluster", "bind_address":"0.0.0.0", "username":"{0}", "password":"{1}", "port": 5984, "node_count":"{2}"  "remote_node": "{3}", "remote_current_user": "{0}", "remote_current_password": "{1}"'.format(creds[0],creds[1],nr_of_peers,name)
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={payload},  auth=creds)
+            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",payload)
+
+            # action: add_node
+            payload = '"action": "add_node", "host":"{0}", "port": 5984, "username": "{1}", "password":"{2}"'.format(name, creds[0],creds[1])
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={payload},  auth=creds)
+            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",payload)
+
+            print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
         if creds[0] and creds[1]:
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
-        else:
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"})
         if (setup_resp.status_code == 201):
             print("CouchDB cluster setup done. Time to relax!")
         else:
@@ -117,6 +127,13 @@ def finish_cluster(names):
     else:
         print("This pod is intentionally skipping the call to http://127.0.0.1:5984/_cluster_setup")
 
+# Run action:enable_cluster on every CouchDB cluster node
+def enable_cluster(nr_of_peers):
+    creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
+    if creds[0] and creds[1]:
+        # http://docs.couchdb.org/en/stable/cluster/setup.html
+        payload = '"action": "enable_cluster", "bind_address":"0.0.0.0", "{0}}": "admin", "{1}}":"password", "node_count":"{2}"'.format(creds[0],creds[1],nr_of_peers)
+        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={payload},  auth=creds)
 
 def sleep_forever():
     while True:
@@ -127,5 +144,9 @@ def sleep_forever():
     print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
     connect_the_dots(peer_names)
     print('Cluster membership populated!')
-    finish_cluster(peer_names)
+    if (os.getenv("COUCHDB_USER") and os.getenv("COUCHDB_PASSWORD")):
+        enable_cluster(len(peer_names))
+        finish_cluster(peer_names)
+    else:
+        print ('Skipping cluster setup. Username and/or password not provided')
     sleep_forever()

From a83865b88bb1f79987f489457e2fdcf397301269 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 15:28:30 +0200
Subject: [PATCH 16/50] Automate _cluster_setup

---
 mem3_helper.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index e597d4b..aa766fc 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -132,8 +132,9 @@ def enable_cluster(nr_of_peers):
     creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
     if creds[0] and creds[1]:
         # http://docs.couchdb.org/en/stable/cluster/setup.html
-        payload = '"action": "enable_cluster", "bind_address":"0.0.0.0", "{0}}": "admin", "{1}}":"password", "node_count":"{2}"'.format(creds[0],creds[1],nr_of_peers)
+        payload = '"action": "enable_cluster", "bind_address":"0.0.0.0", "{0}": "admin", "{1}":"password", "node_count":"{2}"'.format(creds[0],creds[1],nr_of_peers)
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={payload},  auth=creds)
+        print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",payload)
 
 def sleep_forever():
     while True:

From 50de50f733c19df875e982f90fbc5b027bef6d00 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 15:46:09 +0200
Subject: [PATCH 17/50] Automate _cluster_setup

---
 mem3_helper.py | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index aa766fc..186cfc1 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -107,14 +107,28 @@ def finish_cluster(names):
             # http://docs.couchdb.org/en/stable/cluster/setup.html
 
             # action: enable_cluster
-            payload = '"action": "enable_cluster", "bind_address":"0.0.0.0", "username":"{0}", "password":"{1}", "port": 5984, "node_count":"{2}"  "remote_node": "{3}", "remote_current_user": "{0}", "remote_current_password": "{1}"'.format(creds[0],creds[1],nr_of_peers,name)
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={payload},  auth=creds)
-            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",payload)
+            payload = {}
+            payload['action'] = 'enable_cluster'
+            payload['bind_address'] = '0.0.0.0'
+            payload['username'] = creds[0]
+            payload['password'] = creds[1]
+            payload['port'] = 5984
+            payload['node_count'] = len(names)
+            payload['remote_node'] = name
+            payload['remote_current_user'] = creds[0]
+            payload['remote_current_password'] = creds[1]
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds)
+            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
 
             # action: add_node
-            payload = '"action": "add_node", "host":"{0}", "port": 5984, "username": "{1}", "password":"{2}"'.format(name, creds[0],creds[1])
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={payload},  auth=creds)
-            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",payload)
+            payload = {}
+            payload['action'] = 'add_node'
+            payload['username'] = creds[0]
+            payload['password'] = creds[1]
+            payload['port'] = 5984
+            payload['host'] = name
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds)
+            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
 
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!

From efa95bfda1a8b8fb73abc8c04bbf2fd81be32a15 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 15:57:00 +0200
Subject: [PATCH 18/50] Automate _cluster_setup

---
 mem3_helper.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 186cfc1..c2285d6 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -146,9 +146,14 @@ def enable_cluster(nr_of_peers):
     creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
     if creds[0] and creds[1]:
         # http://docs.couchdb.org/en/stable/cluster/setup.html
-        payload = '"action": "enable_cluster", "bind_address":"0.0.0.0", "{0}": "admin", "{1}":"password", "node_count":"{2}"'.format(creds[0],creds[1],nr_of_peers)
-        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={payload},  auth=creds)
-        print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",payload)
+        payload = {}
+        payload['action'] = 'enable_cluster'
+        payload['bind_address'] = '0.0.0.0'
+        payload['username'] = creds[0]
+        payload['password'] = creds[1]
+        payload['node_count'] = len(names)
+        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds)
+        print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
 
 def sleep_forever():
     while True:

From c73cbfae4baaf8f4da95a02d0884dd6f133840b4 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 16:06:52 +0200
Subject: [PATCH 19/50] Automate _cluster_setup

---
 mem3_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index c2285d6..dce4df8 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -151,7 +151,7 @@ def enable_cluster(nr_of_peers):
         payload['bind_address'] = '0.0.0.0'
         payload['username'] = creds[0]
         payload['password'] = creds[1]
-        payload['node_count'] = len(names)
+        payload['node_count'] = nr_of_peers
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds)
         print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
 

From 017fe920f230ad3b6d47d46a95c8cb7e69d086d2 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 16:35:36 +0200
Subject: [PATCH 20/50] Automate _cluster_setup

---
 mem3_helper.py | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index dce4df8..a4651c5 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -106,6 +106,8 @@ def finish_cluster(names):
             # The node in <name> is primed
             # http://docs.couchdb.org/en/stable/cluster/setup.html
 
+            headers = {'Content-type': 'application/json'}
+
             # action: enable_cluster
             payload = {}
             payload['action'] = 'enable_cluster'
@@ -117,7 +119,9 @@ def finish_cluster(names):
             payload['remote_node'] = name
             payload['remote_current_user'] = creds[0]
             payload['remote_current_password'] = creds[1]
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds)
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
+            payload['password'] = "**masked**"
+            payload['remote_current_password'] = "**masked**"
             print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
 
             # action: add_node
@@ -127,7 +131,8 @@ def finish_cluster(names):
             payload['password'] = creds[1]
             payload['port'] = 5984
             payload['host'] = name
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds)
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
+            payload['password'] = "**masked**"
             print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
 
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
@@ -145,14 +150,17 @@ def finish_cluster(names):
 def enable_cluster(nr_of_peers):
     creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
     if creds[0] and creds[1]:
+        headers = {'Content-type': 'application/json'}
+
         # http://docs.couchdb.org/en/stable/cluster/setup.html
         payload = {}
         payload['action'] = 'enable_cluster'
         payload['bind_address'] = '0.0.0.0'
         payload['username'] = creds[0]
         payload['password'] = creds[1]
-        payload['node_count'] = nr_of_peers
-        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds)
+        payload['node_count'] = '{0}'.format(nr_of_peers)
+        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
+        payload['password'] = "**masked**"
         print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
 
 def sleep_forever():

From 5f19920b71fce1ac7841c49bffe7fb6f7094f33c Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 19:00:28 +0200
Subject: [PATCH 21/50] Automate _cluster_setup

---
 mem3_helper.py | 45 +++++++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 20 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index a4651c5..c9974de 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -53,6 +53,24 @@ def connect_the_dots(names):
                 resp = requests.put(uri, data=json.dumps(doc))
         print('Adding CouchDB cluster node', name, "to this pod's CouchDB", flush=True)
 
+# Run action:enable_cluster on every CouchDB cluster node
+def enable_cluster(nr_of_peers):
+    creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
+    if creds[0] and creds[1]:
+        headers = {'Content-type': 'application/json'}
+        print ("== Enabling cluster mode ===")
+        # http://docs.couchdb.org/en/stable/cluster/setup.html
+        payload = {}
+        payload['action'] = 'enable_cluster'
+        payload['bind_address'] = '0.0.0.0'
+        payload['username'] = creds[0]
+        payload['password'] = creds[1]
+        payload['node_count'] = nr_of_peers
+        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
+        payload['password'] = "**masked**"
+        print ("\tRequest: http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+        print ("\tResponse:", setup_resp)
+
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972
 def ordered(obj):
@@ -85,7 +103,7 @@ def finish_cluster(names):
             local_resp = requests.get(local_membership_uri,  auth=creds)
         else:
             local_resp = requests.get(local_membership_uri)
-        # Step through every peer pod and grab the _membership.
+        # Step through every peer. Ensure they are "ready" before progressing.
         for name in names:
             print("Probing {0} for cluster membership".format(name))
             remote_membership_uri = "http://{0}:5984/_membership".format(name)
@@ -103,8 +121,10 @@ def finish_cluster(names):
                 time.sleep(5)
                 if creds[0] and creds[1]:
                     remote_resp = requests.get(remote_membership_uri,  auth=creds)
+            print("Node {0} has all node members in place!".format(name))
             # The node in <name> is primed
             # http://docs.couchdb.org/en/stable/cluster/setup.html
+            print ("== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
 
             headers = {'Content-type': 'application/json'}
 
@@ -122,7 +142,8 @@ def finish_cluster(names):
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
             payload['password'] = "**masked**"
             payload['remote_current_password'] = "**masked**"
-            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
+            print ("\tRequest: http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+            print ("\tResponse:", setup_resp)
 
             # action: add_node
             payload = {}
@@ -133,7 +154,8 @@ def finish_cluster(names):
             payload['host'] = name
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
             payload['password'] = "**masked**"
-            print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
+            print ("\tRequest: http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+            print ("\tResponse:", setup_resp)
 
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
@@ -146,23 +168,6 @@ def finish_cluster(names):
     else:
         print("This pod is intentionally skipping the call to http://127.0.0.1:5984/_cluster_setup")
 
-# Run action:enable_cluster on every CouchDB cluster node
-def enable_cluster(nr_of_peers):
-    creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
-    if creds[0] and creds[1]:
-        headers = {'Content-type': 'application/json'}
-
-        # http://docs.couchdb.org/en/stable/cluster/setup.html
-        payload = {}
-        payload['action'] = 'enable_cluster'
-        payload['bind_address'] = '0.0.0.0'
-        payload['username'] = creds[0]
-        payload['password'] = creds[1]
-        payload['node_count'] = '{0}'.format(nr_of_peers)
-        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
-        payload['password'] = "**masked**"
-        print ("POST to http://127.0.0.1:5984/_cluster_setup returned",setup_resp.status_code,"payload=",json.dumps(payload))
-
 def sleep_forever():
     while True:
         time.sleep(5)

From c7b8f538c6f90b79d5996a4554c7e292ee9b0448 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 20:24:12 +0200
Subject: [PATCH 22/50] Automate _cluster_setup

---
 mem3_helper.py | 39 ++++++++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 11 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index c9974de..ece60d0 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -58,7 +58,7 @@ def enable_cluster(nr_of_peers):
     creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
     if creds[0] and creds[1]:
         headers = {'Content-type': 'application/json'}
-        print ("== Enabling cluster mode ===")
+        print ("=== Enabling cluster mode ===")
         # http://docs.couchdb.org/en/stable/cluster/setup.html
         payload = {}
         payload['action'] = 'enable_cluster'
@@ -68,8 +68,8 @@ def enable_cluster(nr_of_peers):
         payload['node_count'] = nr_of_peers
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
         payload['password'] = "**masked**"
-        print ("\tRequest: http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-        print ("\tResponse:", setup_resp)
+        print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+        print ("\tResponse:", setup_resp.status_code, setup_resp.json())
 
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972
@@ -91,7 +91,6 @@ def finish_cluster(names):
     # https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity
     # we can make sure that this code runs
     # on the "first" pod only with this hack:
-    print('HOSTNAME={0}'.format(os.getenv("HOSTNAME")))
     if (os.getenv("HOSTNAME").endswith("-0")):
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         # Make sure that ALL CouchDB cluster peers have been
@@ -124,7 +123,7 @@ def finish_cluster(names):
             print("Node {0} has all node members in place!".format(name))
             # The node in <name> is primed
             # http://docs.couchdb.org/en/stable/cluster/setup.html
-            print ("== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
+            print ("=== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
 
             headers = {'Content-type': 'application/json'}
 
@@ -142,8 +141,8 @@ def finish_cluster(names):
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
             payload['password'] = "**masked**"
             payload['remote_current_password'] = "**masked**"
-            print ("\tRequest: http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-            print ("\tResponse:", setup_resp)
+            print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
 
             # action: add_node
             payload = {}
@@ -154,15 +153,33 @@ def finish_cluster(names):
             payload['host'] = name
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
             payload['password'] = "**masked**"
-            print ("\tRequest: http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-            print ("\tResponse:", setup_resp)
+            print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
 
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
-        if creds[0] and creds[1]:
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
+        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         if (setup_resp.status_code == 201):
+            print("== CouchDB cluster setup done! ===")
+            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup")
             print("CouchDB cluster setup done. Time to relax!")
+            print ("\tRequest: GET http://127.0.0.1:5984/_cluster_setup")
+
+            print("== Creating default databases ===")
+
+            setup_resp=requests.put("http://127.0.0.1:5984/_users",  auth=creds, headers=headers)
+            print ("\tRequest: PUT http://127.0.0.1:5984/_users")
+            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+
+            setup_resp=requests.put("http://127.0.0.1:5984/_replicator",  auth=creds, headers=headers)
+            print ("\tRequest: PUT http://127.0.0.1:5984/_replicator")
+            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+
+            setup_resp=requests.put("http://127.0.0.1:5984/_global_changes",  auth=creds, headers=headers)
+            print ("\tRequest: PUT http://127.0.0.1:5984/_global_changes")
+            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+
+            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
         else:
             print('Ouch! Failed the final step: http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
     else:

From fd53bf19a587f453d684315a253961fc689ba98c Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 21:14:37 +0200
Subject: [PATCH 23/50] Automate _cluster_setup

---
 mem3_helper.py | 84 ++++++++++++++++++++++++++------------------------
 1 file changed, 43 insertions(+), 41 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index ece60d0..703f9f5 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -69,7 +69,7 @@ def enable_cluster(nr_of_peers):
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
         payload['password'] = "**masked**"
         print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-        print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
 
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972
@@ -93,6 +93,41 @@ def finish_cluster(names):
     # on the "first" pod only with this hack:
     if (os.getenv("HOSTNAME").endswith("-0")):
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
+
+        headers = {'Content-type': 'application/json'}
+        print ("=== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
+        for name in names:
+            # Exclude "this" pod
+            if (name.split(".", 1) != os.getenv("HOSTNAME")):
+                # action: enable_cluster
+                payload = {}
+                payload['action'] = 'enable_cluster'
+                payload['bind_address'] = '0.0.0.0'
+                payload['username'] = creds[0]
+                payload['password'] = creds[1]
+                payload['port'] = 5984
+                payload['node_count'] = len(names)
+                payload['remote_node'] = name
+                payload['remote_current_user'] = creds[0]
+                payload['remote_current_password'] = creds[1]
+                setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
+                payload['password'] = "**masked**"
+                payload['remote_current_password'] = "**masked**"
+                print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+                print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+
+                # action: add_node
+                payload = {}
+                payload['action'] = 'add_node'
+                payload['username'] = creds[0]
+                payload['password'] = creds[1]
+                payload['port'] = 5984
+                payload['host'] = name
+                setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
+                payload['password'] = "**masked**"
+                print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
+                print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+
         # Make sure that ALL CouchDB cluster peers have been
         # primed with _nodes data before /_cluster_setup
         # Use the _membership of "this" pod's CouchDB as reference
@@ -121,40 +156,7 @@ def finish_cluster(names):
                 if creds[0] and creds[1]:
                     remote_resp = requests.get(remote_membership_uri,  auth=creds)
             print("Node {0} has all node members in place!".format(name))
-            # The node in <name> is primed
-            # http://docs.couchdb.org/en/stable/cluster/setup.html
-            print ("=== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
-
-            headers = {'Content-type': 'application/json'}
-
-            # action: enable_cluster
-            payload = {}
-            payload['action'] = 'enable_cluster'
-            payload['bind_address'] = '0.0.0.0'
-            payload['username'] = creds[0]
-            payload['password'] = creds[1]
-            payload['port'] = 5984
-            payload['node_count'] = len(names)
-            payload['remote_node'] = name
-            payload['remote_current_user'] = creds[0]
-            payload['remote_current_password'] = creds[1]
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
-            payload['password'] = "**masked**"
-            payload['remote_current_password'] = "**masked**"
-            print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
-
-            # action: add_node
-            payload = {}
-            payload['action'] = 'add_node'
-            payload['username'] = creds[0]
-            payload['password'] = creds[1]
-            payload['port'] = 5984
-            payload['host'] = name
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
-            payload['password'] = "**masked**"
-            print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+
 
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
@@ -162,24 +164,24 @@ def finish_cluster(names):
         if (setup_resp.status_code == 201):
             print("== CouchDB cluster setup done! ===")
             setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup")
-            print("CouchDB cluster setup done. Time to relax!")
+            print("CouchDB cluster setup done.")
             print ("\tRequest: GET http://127.0.0.1:5984/_cluster_setup")
-
+            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
             print("== Creating default databases ===")
 
             setup_resp=requests.put("http://127.0.0.1:5984/_users",  auth=creds, headers=headers)
             print ("\tRequest: PUT http://127.0.0.1:5984/_users")
-            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
 
             setup_resp=requests.put("http://127.0.0.1:5984/_replicator",  auth=creds, headers=headers)
             print ("\tRequest: PUT http://127.0.0.1:5984/_replicator")
-            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
 
             setup_resp=requests.put("http://127.0.0.1:5984/_global_changes",  auth=creds, headers=headers)
             print ("\tRequest: PUT http://127.0.0.1:5984/_global_changes")
-            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
 
-            print ("\tResponse:", setup_resp.status_code, setup_resp.json())
+            print("Time to relax!")
         else:
             print('Ouch! Failed the final step: http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
     else:

From 3ad17e11d8c933ce5bad2bb06f10efbb851b8e61 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 22:26:44 +0200
Subject: [PATCH 24/50] Automate _cluster_setup

---
 mem3_helper.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 703f9f5..68eeced 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -98,7 +98,7 @@ def finish_cluster(names):
         print ("=== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
         for name in names:
             # Exclude "this" pod
-            if (name.split(".", 1) != os.getenv("HOSTNAME")):
+            if (name.split(".",1)[0]) != os.getenv("HOSTNAME")):
                 # action: enable_cluster
                 payload = {}
                 payload['action'] = 'enable_cluster'
@@ -194,6 +194,8 @@ def sleep_forever():
 if __name__ == '__main__':
     peer_names = discover_peers(construct_service_record())
     print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
+    if (os.getenv("COUCHDB_USER") and os.getenv("COUCHDB_PASSWORD")):
+        enable_cluster(len(peer_names))
     connect_the_dots(peer_names)
     print('Cluster membership populated!')
     if (os.getenv("COUCHDB_USER") and os.getenv("COUCHDB_PASSWORD")):

From 31bc09435612af258688e4ccb61c6cc468f49088 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 22:51:49 +0200
Subject: [PATCH 25/50] Automate _cluster_setup

---
 mem3_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 68eeced..4eaf9e4 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -98,7 +98,7 @@ def finish_cluster(names):
         print ("=== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
         for name in names:
             # Exclude "this" pod
-            if (name.split(".",1)[0]) != os.getenv("HOSTNAME")):
+            if (name.split(".",1)[0] != os.getenv("HOSTNAME")):
                 # action: enable_cluster
                 payload = {}
                 payload['action'] = 'enable_cluster'

From b626efdeb2bd5ff6c41b80df315edbb77efa453d Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 23:31:17 +0200
Subject: [PATCH 26/50] Automate _cluster_setup

---
 mem3_helper.py | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 4eaf9e4..9a379f7 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -157,28 +157,29 @@ def finish_cluster(names):
                     remote_resp = requests.get(remote_membership_uri,  auth=creds)
             print("Node {0} has all node members in place!".format(name))
 
-
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
+
+        print("== Creating default databases ===")
+        setup_resp=requests.put("http://127.0.0.1:5984/_users",  auth=creds)
+        print ("\tRequest: PUT http://127.0.0.1:5984/_users")
+        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+
+        setup_resp=requests.put("http://127.0.0.1:5984/_replicator",  auth=creds)
+        print ("\tRequest: PUT http://127.0.0.1:5984/_replicator")
+        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+
+        setup_resp=requests.put("http://127.0.0.1:5984/_global_changes",  auth=creds)
+        print ("\tRequest: PUT http://127.0.0.1:5984/_global_changes")
+        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         if (setup_resp.status_code == 201):
             print("== CouchDB cluster setup done! ===")
-            setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup")
-            print("CouchDB cluster setup done.")
-            print ("\tRequest: GET http://127.0.0.1:5984/_cluster_setup")
-            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-            print("== Creating default databases ===")
-
-            setup_resp=requests.put("http://127.0.0.1:5984/_users",  auth=creds, headers=headers)
-            print ("\tRequest: PUT http://127.0.0.1:5984/_users")
+            print ('\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"}')
             print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
-            setup_resp=requests.put("http://127.0.0.1:5984/_replicator",  auth=creds, headers=headers)
-            print ("\tRequest: PUT http://127.0.0.1:5984/_replicator")
-            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
-            setup_resp=requests.put("http://127.0.0.1:5984/_global_changes",  auth=creds, headers=headers)
-            print ("\tRequest: PUT http://127.0.0.1:5984/_global_changes")
+            setup_resp=requests.get("http://127.0.0.1:5984/_cluster_setup",  auth=creds)
+            print ('\tRequest: GET http://127.0.0.1:5984/_cluster_setup')
             print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
 
             print("Time to relax!")
@@ -199,7 +200,6 @@ def sleep_forever():
     connect_the_dots(peer_names)
     print('Cluster membership populated!')
     if (os.getenv("COUCHDB_USER") and os.getenv("COUCHDB_PASSWORD")):
-        enable_cluster(len(peer_names))
         finish_cluster(peer_names)
     else:
         print ('Skipping cluster setup. Username and/or password not provided')

From 54d60202b454ca340d586344e414c210ffe7d9be Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Thu, 23 Aug 2018 23:52:10 +0200
Subject: [PATCH 27/50] Automate _cluster_setup

---
 mem3_helper.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 9a379f7..b1c1f07 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -160,7 +160,7 @@ def finish_cluster(names):
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
 
-        print("== Creating default databases ===")
+"""         print("== Creating default databases ===")
         setup_resp=requests.put("http://127.0.0.1:5984/_users",  auth=creds)
         print ("\tRequest: PUT http://127.0.0.1:5984/_users")
         print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
@@ -171,7 +171,7 @@ def finish_cluster(names):
 
         setup_resp=requests.put("http://127.0.0.1:5984/_global_changes",  auth=creds)
         print ("\tRequest: PUT http://127.0.0.1:5984/_global_changes")
-        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json()) """
 
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         if (setup_resp.status_code == 201):

From 76e116cf547180ba5ecc0a60629d364f0c4473f7 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 00:00:07 +0200
Subject: [PATCH 28/50] Automate _cluster_setup

---
 mem3_helper.py | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index b1c1f07..25dbb41 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -160,19 +160,6 @@ def finish_cluster(names):
             print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
         # At this point ALL peers have _nodes populated. Finish the cluster setup!
 
-"""         print("== Creating default databases ===")
-        setup_resp=requests.put("http://127.0.0.1:5984/_users",  auth=creds)
-        print ("\tRequest: PUT http://127.0.0.1:5984/_users")
-        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
-        setup_resp=requests.put("http://127.0.0.1:5984/_replicator",  auth=creds)
-        print ("\tRequest: PUT http://127.0.0.1:5984/_replicator")
-        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
-        setup_resp=requests.put("http://127.0.0.1:5984/_global_changes",  auth=creds)
-        print ("\tRequest: PUT http://127.0.0.1:5984/_global_changes")
-        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json()) """
-
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         if (setup_resp.status_code == 201):
             print("== CouchDB cluster setup done! ===")

From 4a8968513f7aec0065f4cbf13f59a6f6ae3c017e Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 10:45:50 +0200
Subject: [PATCH 29/50] Automate _cluster_setup

---
 mem3_helper.py | 89 ++++++++++++++++++++++++++++++--------------------
 1 file changed, 53 insertions(+), 36 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 25dbb41..93a6558 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -51,7 +51,7 @@ def connect_the_dots(names):
                 resp = requests.put(uri, data=json.dumps(doc), auth=creds)
             else:
                 resp = requests.put(uri, data=json.dumps(doc))
-        print('Adding CouchDB cluster node', name, "to this pod's CouchDB", flush=True)
+        print('Adding CouchDB cluster node', name, "to this pod's CouchDB. Response code:", resp.status_code ,flush=True)
 
 # Run action:enable_cluster on every CouchDB cluster node
 def enable_cluster(nr_of_peers):
@@ -128,37 +128,7 @@ def finish_cluster(names):
                 print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
                 print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
 
-        # Make sure that ALL CouchDB cluster peers have been
-        # primed with _nodes data before /_cluster_setup
-        # Use the _membership of "this" pod's CouchDB as reference
-        local_membership_uri = "http://127.0.0.1:5984/_membership"
-        print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
-        if creds[0] and creds[1]:
-            local_resp = requests.get(local_membership_uri,  auth=creds)
-        else:
-            local_resp = requests.get(local_membership_uri)
-        # Step through every peer. Ensure they are "ready" before progressing.
-        for name in names:
-            print("Probing {0} for cluster membership".format(name))
-            remote_membership_uri = "http://{0}:5984/_membership".format(name)
-            if creds[0] and creds[1]:
-                remote_resp = requests.get(remote_membership_uri,  auth=creds)
-            # Compare local and remote _mebership data. Make sure the set
-            # of nodes match. This will ensure that the remote nodes
-            # are fully primed with nodes data before progressing with
-            # _cluster_setup
-            while (remote_resp.status_code != 200) or (ordered(local_resp.json()) != ordered(remote_resp.json())):
-                # print ("remote_resp.status_code",remote_resp.status_code)
-                # print (ordered(local_resp.json()))
-                # print (ordered(remote_resp.json()))
-                print('Waiting for node {0} to have all node members populated'.format(name),flush=True)
-                time.sleep(5)
-                if creds[0] and creds[1]:
-                    remote_resp = requests.get(remote_membership_uri,  auth=creds)
-            print("Node {0} has all node members in place!".format(name))
-
-            print('CouchDB cluster peer {} added to "setup coordination node"'.format(name))
-        # At this point ALL peers have _nodes populated. Finish the cluster setup!
+
 
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         if (setup_resp.status_code == 201):
@@ -175,19 +145,66 @@ def finish_cluster(names):
     else:
         print("This pod is intentionally skipping the call to http://127.0.0.1:5984/_cluster_setup")
 
+def diff(first, second):
+        second = set(second)
+        return [item for item in first if item not in second]
+
+# Check if the _membership API on all (known) CouchDB nodes have the same values.
+# Returns true if sam. False in any other situation.
+def are_nodes_in_sync(names):
+    # Make sure that ALL (known) CouchDB cluster peers have been
+    # have the same _membership data.Use "this" nodes memebership as
+    # "source"
+    local_membership_uri = "http://127.0.0.1:5984/_membership"
+    print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
+    if creds[0] and creds[1]:
+        local_resp = requests.get(local_membership_uri,  auth=creds)
+    else:
+        local_resp = requests.get(local_membership_uri)
+
+    # If any difference is found - set to true
+    is_different = False;
+    # Step through every peer. Ensure they are "ready" before progressing.
+    for name in names:
+        print("Probing {0} for cluster membership".format(name))
+        remote_membership_uri = "http://{0}:5984/_membership".format(name)
+        if creds[0] and creds[1]:
+            remote_resp = requests.get(remote_membership_uri,  auth=creds)
+        # Compare local and remote _mebership data. Make sure the set
+        # of nodes match. This will ensure that the remote nodes
+        # are fully primed with nodes data before progressing with
+        # _cluster_setup
+        if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
+            if ordered(local_resp.json()) != ordered(remote_resp.json()):
+                is_different = True
+                print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
+                records_in_local_but_not_in_remote = diff(local_resp.json().cluster_nodes, remote_resp.json().cluster_nodes)
+                records_in_remote_but_not_in_local = diff(remote_resp.json().cluster_nodes, local_resp.json().cluster_nodes)
+                if records_in_local_but_not_in_remote:
+                    print ("Cluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
+                if records_in_remote_but_not_in_local:
+                    print ("Cluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
+        else:
+            is_different = True
+    return not is_different
+
 def sleep_forever():
     while True:
         time.sleep(5)
 
 if __name__ == '__main__':
     peer_names = discover_peers(construct_service_record())
-    print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
-    if (os.getenv("COUCHDB_USER") and os.getenv("COUCHDB_PASSWORD")):
-        enable_cluster(len(peer_names))
     connect_the_dots(peer_names)
+    print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
+    # loop until all CouchDB nodes discovered
+    while not are_nodes_in_sync(names):
+        time.sleep(5)
+        peer_names = discover_peers(construct_service_record())
+        connect_the_dots(peer_names)
     print('Cluster membership populated!')
+    
     if (os.getenv("COUCHDB_USER") and os.getenv("COUCHDB_PASSWORD")):
         finish_cluster(peer_names)
     else:
-        print ('Skipping cluster setup. Username and/or password not provided')
+        print ('Skipping cluster final setup. Username and/or password not provided')
     sleep_forever()

From b3c7328f252e1a93022495786093b898c2027bd9 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 12:04:10 +0200
Subject: [PATCH 30/50] Automate _cluster_setup

---
 mem3_helper.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 93a6558..32273c4 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -149,6 +149,12 @@ def diff(first, second):
         second = set(second)
         return [item for item in first if item not in second]
 
+
+@backoff.on_exception(
+    backoff.expo,
+    requests.exceptions.ConnectionError,
+    max_tries=10
+)
 # Check if the _membership API on all (known) CouchDB nodes have the same values.
 # Returns true if sam. False in any other situation.
 def are_nodes_in_sync(names):
@@ -197,7 +203,7 @@ def sleep_forever():
     connect_the_dots(peer_names)
     print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
     # loop until all CouchDB nodes discovered
-    while not are_nodes_in_sync(names):
+    while not are_nodes_in_sync(peer_names):
         time.sleep(5)
         peer_names = discover_peers(construct_service_record())
         connect_the_dots(peer_names)

From 6c9bbd77146adb235ae280181c93a8a78e0cefda Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 12:34:47 +0200
Subject: [PATCH 31/50] Automate _cluster_setup

---
 mem3_helper.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 32273c4..645cc27 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -44,14 +44,15 @@ def connect_the_dots(names):
             resp = requests.put(uri, data=json.dumps(doc), auth=creds)
         else:
             resp = requests.put(uri, data=json.dumps(doc))
-        while resp.status_code != 201:
-            print('Waiting for _nodes DB to be created.',uri,'returned', resp.status_code, flush=True)
+        while resp.status_code != 201 and resp.status_code != 409:
+            print('Waiting for _nodes DB to be created.',uri,'returned', resp.status_code, resp.json(), flush=True)
             time.sleep(5)
             if creds[0] and creds[1]:
                 resp = requests.put(uri, data=json.dumps(doc), auth=creds)
             else:
                 resp = requests.put(uri, data=json.dumps(doc))
-        print('Adding CouchDB cluster node', name, "to this pod's CouchDB. Response code:", resp.status_code ,flush=True)
+        if resp.status_code == 201:
+            print('Adding CouchDB cluster node', name, "to this pod's CouchDB. Response code:", resp.status_code ,flush=True)
 
 # Run action:enable_cluster on every CouchDB cluster node
 def enable_cluster(nr_of_peers):

From efba00b677313b5021fc2cc45563f0ee84254998 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 13:03:26 +0200
Subject: [PATCH 32/50] Automate _cluster_setup

---
 mem3_helper.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/mem3_helper.py b/mem3_helper.py
index 645cc27..5d5b7fc 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -164,6 +164,7 @@ def are_nodes_in_sync(names):
     # "source"
     local_membership_uri = "http://127.0.0.1:5984/_membership"
     print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
+    creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
     if creds[0] and creds[1]:
         local_resp = requests.get(local_membership_uri,  auth=creds)
     else:
@@ -182,6 +183,9 @@ def are_nodes_in_sync(names):
         # are fully primed with nodes data before progressing with
         # _cluster_setup
         if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
+            if len(local_resp.json()) < 2:
+                # Minimum 2 nodes to form cluster!
+                is_different = True
             if ordered(local_resp.json()) != ordered(remote_resp.json()):
                 is_different = True
                 print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)

From 0785207fda74f3bd8eeeade316461ec09269d006 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 13:34:51 +0200
Subject: [PATCH 33/50] Automate _cluster_setup

---
 mem3_helper.py | 71 +++++++-------------------------------------------
 1 file changed, 10 insertions(+), 61 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 5d5b7fc..2c284f1 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -3,6 +3,9 @@
 This script grabs the list of pod names behind the headless service
 associated with our StatefulSet and feeds those as `couchdb@FQDN`
 nodes to mem3.
+
+As a final step - on just one of the nodes - the cluster finalize is triggered.
+For that part to happen we need username and password.
 """
 
 import json
@@ -54,24 +57,6 @@ def connect_the_dots(names):
         if resp.status_code == 201:
             print('Adding CouchDB cluster node', name, "to this pod's CouchDB. Response code:", resp.status_code ,flush=True)
 
-# Run action:enable_cluster on every CouchDB cluster node
-def enable_cluster(nr_of_peers):
-    creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
-    if creds[0] and creds[1]:
-        headers = {'Content-type': 'application/json'}
-        print ("=== Enabling cluster mode ===")
-        # http://docs.couchdb.org/en/stable/cluster/setup.html
-        payload = {}
-        payload['action'] = 'enable_cluster'
-        payload['bind_address'] = '0.0.0.0'
-        payload['username'] = creds[0]
-        payload['password'] = creds[1]
-        payload['node_count'] = nr_of_peers
-        setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
-        payload['password'] = "**masked**"
-        print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972
 def ordered(obj):
@@ -82,7 +67,7 @@ def ordered(obj):
     else:
         return obj
 
-# Run action:finish_cluster on one and only one CouchDB cluster node
+# Run action:finish_cluster on one (and only one) CouchDB cluster node
 def finish_cluster(names):
     # The HTTP POST to /_cluster_setup should be done to
     # one (and only one) of the CouchDB cluster nodes.
@@ -94,52 +79,15 @@ def finish_cluster(names):
     # on the "first" pod only with this hack:
     if (os.getenv("HOSTNAME").endswith("-0")):
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
-
-        headers = {'Content-type': 'application/json'}
-        print ("=== Adding nodes to CouchDB cluster via the “setup coordination node” ===")
-        for name in names:
-            # Exclude "this" pod
-            if (name.split(".",1)[0] != os.getenv("HOSTNAME")):
-                # action: enable_cluster
-                payload = {}
-                payload['action'] = 'enable_cluster'
-                payload['bind_address'] = '0.0.0.0'
-                payload['username'] = creds[0]
-                payload['password'] = creds[1]
-                payload['port'] = 5984
-                payload['node_count'] = len(names)
-                payload['remote_node'] = name
-                payload['remote_current_user'] = creds[0]
-                payload['remote_current_password'] = creds[1]
-                setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
-                payload['password'] = "**masked**"
-                payload['remote_current_password'] = "**masked**"
-                print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-                print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
-                # action: add_node
-                payload = {}
-                payload['action'] = 'add_node'
-                payload['username'] = creds[0]
-                payload['password'] = creds[1]
-                payload['port'] = 5984
-                payload['host'] = name
-                setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json.dumps(payload),  auth=creds, headers=headers)
-                payload['password'] = "**masked**"
-                print ("\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload:",json.dumps(payload))
-                print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
-
-
+        print("== Get the cluster up and running ===")
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
+        print ('\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"}')
+        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
         if (setup_resp.status_code == 201):
-            print("== CouchDB cluster setup done! ===")
-            print ('\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"}')
-            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+            print ("\tSweet! Just a final check for the logs...")
             setup_resp=requests.get("http://127.0.0.1:5984/_cluster_setup",  auth=creds)
             print ('\tRequest: GET http://127.0.0.1:5984/_cluster_setup')
             print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
-
             print("Time to relax!")
         else:
             print('Ouch! Failed the final step: http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
@@ -157,7 +105,7 @@ def diff(first, second):
     max_tries=10
 )
 # Check if the _membership API on all (known) CouchDB nodes have the same values.
-# Returns true if sam. False in any other situation.
+# Returns true if same. False in any other situation.
 def are_nodes_in_sync(names):
     # Make sure that ALL (known) CouchDB cluster peers have been
     # have the same _membership data.Use "this" nodes memebership as
@@ -207,6 +155,7 @@ def sleep_forever():
     peer_names = discover_peers(construct_service_record())
     connect_the_dots(peer_names)
     print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
+
     # loop until all CouchDB nodes discovered
     while not are_nodes_in_sync(peer_names):
         time.sleep(5)

From ff894dd870a80236aa8c9e3123c157de620ace76 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 14:20:10 +0200
Subject: [PATCH 34/50] Automate _cluster_setup

---
 mem3_helper.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 2c284f1..bd10e65 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -94,11 +94,6 @@ def finish_cluster(names):
     else:
         print("This pod is intentionally skipping the call to http://127.0.0.1:5984/_cluster_setup")
 
-def diff(first, second):
-        second = set(second)
-        return [item for item in first if item not in second]
-
-
 @backoff.on_exception(
     backoff.expo,
     requests.exceptions.ConnectionError,
@@ -126,6 +121,8 @@ def are_nodes_in_sync(names):
         remote_membership_uri = "http://{0}:5984/_membership".format(name)
         if creds[0] and creds[1]:
             remote_resp = requests.get(remote_membership_uri,  auth=creds)
+        else:
+            remote_resp = requests.get(remote_membership_uri)
         # Compare local and remote _mebership data. Make sure the set
         # of nodes match. This will ensure that the remote nodes
         # are fully primed with nodes data before progressing with
@@ -137,14 +134,15 @@ def are_nodes_in_sync(names):
             if ordered(local_resp.json()) != ordered(remote_resp.json()):
                 is_different = True
                 print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
-                records_in_local_but_not_in_remote = diff(local_resp.json().cluster_nodes, remote_resp.json().cluster_nodes)
-                records_in_remote_but_not_in_local = diff(remote_resp.json().cluster_nodes, local_resp.json().cluster_nodes)
+                records_in_local_but_not_in_remote = set(local_resp.json().cluster_nodes) - set(remote_resp.json().cluster_nodes)
+                records_in_remote_but_not_in_local = set(remote_resp.json().cluster_nodes) - set(local_resp.json().cluster_nodes)
                 if records_in_local_but_not_in_remote:
                     print ("Cluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
                 if records_in_remote_but_not_in_local:
                     print ("Cluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
         else:
             is_different = True
+        print('returnerar', not is_different)
     return not is_different
 
 def sleep_forever():

From f192f5794f38a987683517660301907359c453f9 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 14:52:36 +0200
Subject: [PATCH 35/50] Automate _cluster_setup

---
 mem3_helper.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index bd10e65..fbdd3dc 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -114,7 +114,8 @@ def are_nodes_in_sync(names):
         local_resp = requests.get(local_membership_uri)
 
     # If any difference is found - set to true
-    is_different = False;
+    is_different = False
+
     # Step through every peer. Ensure they are "ready" before progressing.
     for name in names:
         print("Probing {0} for cluster membership".format(name))
@@ -123,6 +124,11 @@ def are_nodes_in_sync(names):
             remote_resp = requests.get(remote_membership_uri,  auth=creds)
         else:
             remote_resp = requests.get(remote_membership_uri)
+
+        if len(names) < 2:
+            # Minimum 2 nodes to form cluster!
+            is_different = True   
+
         # Compare local and remote _mebership data. Make sure the set
         # of nodes match. This will ensure that the remote nodes
         # are fully primed with nodes data before progressing with
@@ -142,6 +148,10 @@ def are_nodes_in_sync(names):
                     print ("Cluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
         else:
             is_different = True
+ 
+        if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
+            print("local: ",local_resp.json().cluster_nodes)
+            print("remote: ",remote_resp.json().cluster_nodes)
         print('returnerar', not is_different)
     return not is_different
 

From 2c9a42ba3bdb3bf25f6b488cf6f9136efdf0d6e1 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 15:16:08 +0200
Subject: [PATCH 36/50] Automate _cluster_setup

---
 mem3_helper.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index fbdd3dc..fac11b2 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -140,8 +140,8 @@ def are_nodes_in_sync(names):
             if ordered(local_resp.json()) != ordered(remote_resp.json()):
                 is_different = True
                 print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
-                records_in_local_but_not_in_remote = set(local_resp.json().cluster_nodes) - set(remote_resp.json().cluster_nodes)
-                records_in_remote_but_not_in_local = set(remote_resp.json().cluster_nodes) - set(local_resp.json().cluster_nodes)
+                records_in_local_but_not_in_remote = set(local_resp.json()['cluster_nodes']) - set(remote_resp.json()['cluster_nodes'])
+                records_in_remote_but_not_in_local = set(remote_resp.json()['cluster_nodes']) - set(local_resp.json()['cluster_nodes'])
                 if records_in_local_but_not_in_remote:
                     print ("Cluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
                 if records_in_remote_but_not_in_local:
@@ -150,8 +150,8 @@ def are_nodes_in_sync(names):
             is_different = True
  
         if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
-            print("local: ",local_resp.json().cluster_nodes)
-            print("remote: ",remote_resp.json().cluster_nodes)
+            print("local: ",local_resp.json()['cluster_nodes'])
+            print("remote: ",remote_resp.json()['cluster_nodes'])
         print('returnerar', not is_different)
     return not is_different
 

From 08349160fa15b4c0628e58d24f3f887277d47241 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 15:38:55 +0200
Subject: [PATCH 37/50] Automate _cluster_setup

---
 mem3_helper.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index fac11b2..2d13d34 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -67,7 +67,7 @@ def ordered(obj):
     else:
         return obj
 
-# Run action:finish_cluster on one (and only one) CouchDB cluster node
+# Call action:finish_cluster on one (and only one) CouchDB cluster node
 def finish_cluster(names):
     # The HTTP POST to /_cluster_setup should be done to
     # one (and only one) of the CouchDB cluster nodes.
@@ -90,9 +90,10 @@ def finish_cluster(names):
             print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
             print("Time to relax!")
         else:
-            print('Ouch! Failed the final step: http://127.0.0.1:5984/_cluster_setup returned {0}'.format(setup_resp.status_code))
+            print('Ouch! Failed the final step finalizing the cluster.')
     else:
-        print("This pod is intentionally skipping the call to http://127.0.0.1:5984/_cluster_setup")
+        print('This pod is intentionally skipping the POST to http://127.0.0.1:5984/_cluster_setup {"action": "finish_cluster"}')
+
 
 @backoff.on_exception(
     backoff.expo,
@@ -106,7 +107,6 @@ def are_nodes_in_sync(names):
     # have the same _membership data.Use "this" nodes memebership as
     # "source"
     local_membership_uri = "http://127.0.0.1:5984/_membership"
-    print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
     creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))   
     if creds[0] and creds[1]:
         local_resp = requests.get(local_membership_uri,  auth=creds)
@@ -139,20 +139,16 @@ def are_nodes_in_sync(names):
                 is_different = True
             if ordered(local_resp.json()) != ordered(remote_resp.json()):
                 is_different = True
-                print ("Fetching CouchDB node mebership from this pod: {0}".format(local_membership_uri),flush=True)
+
+                # For logging only...
                 records_in_local_but_not_in_remote = set(local_resp.json()['cluster_nodes']) - set(remote_resp.json()['cluster_nodes'])
-                records_in_remote_but_not_in_local = set(remote_resp.json()['cluster_nodes']) - set(local_resp.json()['cluster_nodes'])
                 if records_in_local_but_not_in_remote:
-                    print ("Cluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
+                    print ("\tCluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
+                records_in_remote_but_not_in_local = set(remote_resp.json()['cluster_nodes']) - set(local_resp.json()['cluster_nodes'])
                 if records_in_remote_but_not_in_local:
-                    print ("Cluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
+                    print ("\tCluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
         else:
             is_different = True
- 
-        if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
-            print("local: ",local_resp.json()['cluster_nodes'])
-            print("remote: ",remote_resp.json()['cluster_nodes'])
-        print('returnerar', not is_different)
     return not is_different
 
 def sleep_forever():

From 00d1954df51a9b822af0458bf2148c80064e852a Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 16:03:21 +0200
Subject: [PATCH 38/50] Automate _cluster_setup

---
 mem3_helper.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 2d13d34..2b6f009 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -114,7 +114,7 @@ def are_nodes_in_sync(names):
         local_resp = requests.get(local_membership_uri)
 
     # If any difference is found - set to true
-    is_different = False
+    not_ready = False
 
     # Step through every peer. Ensure they are "ready" before progressing.
     for name in names:
@@ -127,18 +127,20 @@ def are_nodes_in_sync(names):
 
         if len(names) < 2:
             # Minimum 2 nodes to form cluster!
-            is_different = True   
+            not_ready = True
+            print("\tNeed at least 2 DNS records to start with. Got ",len(names))
 
         # Compare local and remote _mebership data. Make sure the set
         # of nodes match. This will ensure that the remote nodes
         # are fully primed with nodes data before progressing with
         # _cluster_setup
         if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
-            if len(local_resp.json()) < 2:
+            if len(local_resp.json()['cluster_nodes']) < 2:
                 # Minimum 2 nodes to form cluster!
-                is_different = True
+                not_ready = True
+                print("\tNeed at least 2 cluster nodes in the _membership of pod",os.getenv("HOSTNAME"))
             if ordered(local_resp.json()) != ordered(remote_resp.json()):
-                is_different = True
+                not_ready = True
 
                 # For logging only...
                 records_in_local_but_not_in_remote = set(local_resp.json()['cluster_nodes']) - set(remote_resp.json()['cluster_nodes'])
@@ -148,8 +150,8 @@ def are_nodes_in_sync(names):
                 if records_in_remote_but_not_in_local:
                     print ("\tCluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
         else:
-            is_different = True
-    return not is_different
+            not_ready = True
+    return not not_ready
 
 def sleep_forever():
     while True:

From f3eed40bb2be1ee7f1c92aeebb3c382d60e4b211 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Fri, 24 Aug 2018 16:17:47 +0200
Subject: [PATCH 39/50] Automate _cluster_setup

---
 mem3_helper.py | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 2b6f009..b720b69 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -31,7 +31,9 @@ def discover_peers(service_record):
     # Erlang requires that we drop the trailing period from the absolute DNS
     # name to form the hostname used for the Erlang node. This feels hacky
     # but not sure of a more official answer
-    return [rdata.target.to_text()[:-1] for rdata in answers]
+    result = [rdata.target.to_text()[:-1] for rdata in answers]
+    print("\tGot the following peers' fqdm from DNS lookup:",result,flush=True)
+    return result
 
 @backoff.on_exception(
     backoff.expo,
@@ -135,20 +137,23 @@ def are_nodes_in_sync(names):
         # are fully primed with nodes data before progressing with
         # _cluster_setup
         if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
-            if len(local_resp.json()['cluster_nodes']) < 2:
-                # Minimum 2 nodes to form cluster!
-                not_ready = True
-                print("\tNeed at least 2 cluster nodes in the _membership of pod",os.getenv("HOSTNAME"))
-            if ordered(local_resp.json()) != ordered(remote_resp.json()):
+            if ordered(local_resp.json()) == ordered(remote_resp.json()):
+                print ("\tIn sync!")
+            else:
                 not_ready = True
-
-                # For logging only...
+                # Rest is for logging only...
                 records_in_local_but_not_in_remote = set(local_resp.json()['cluster_nodes']) - set(remote_resp.json()['cluster_nodes'])
                 if records_in_local_but_not_in_remote:
                     print ("\tCluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
                 records_in_remote_but_not_in_local = set(remote_resp.json()['cluster_nodes']) - set(local_resp.json()['cluster_nodes'])
                 if records_in_remote_but_not_in_local:
                     print ("\tCluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
+
+            # Cover the case where local pod has 1 record only
+            if len(local_resp.json()['cluster_nodes']) < 2:
+                # Minimum 2 nodes to form cluster!
+                not_ready = True
+                print("\tNeed at least 2 cluster nodes in the _membership of pod",os.getenv("HOSTNAME"))
         else:
             not_ready = True
     return not not_ready
@@ -160,7 +165,6 @@ def sleep_forever():
 if __name__ == '__main__':
     peer_names = discover_peers(construct_service_record())
     connect_the_dots(peer_names)
-    print("Got the following peers' fqdm from DNS lookup:",peer_names,flush=True)
 
     # loop until all CouchDB nodes discovered
     while not are_nodes_in_sync(peer_names):

From e1b27e807022bc7d2f1aac8e900d2f05018afd9f Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 09:14:12 +0200
Subject: [PATCH 40/50] Automate _cluster_setup

---
 mem3_helper.py | 54 +++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index b720b69..36ea0a7 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -41,23 +41,53 @@ def discover_peers(service_record):
     max_tries=10
 )
 def connect_the_dots(names):
-    creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
+
+    # Ordinal Index: For a StatefulSet with N replicas, each Pod in the StatefulSet
+    # will be assigned an integer ordinal, from 0 up through N-1, that is unique over the Set.
+    # Approach: get the Ordinal index of this pod and make sure that the list of name
+    # include all those ordinals.
+    # By looking at this pods Ordinal, make sure that all DNS records for
+    # pods having lesser Ordinal are found before adding any nodes to CouchDB.
+    # This is done to PREVENT the following case:
+    # (1) POD with ordnial 1 get DNS records for ordinal 1 and 2.
+    # (2) POD with ordinal 2 get DNS records for ordinal 1 and 2.
+    # (3) The are_nodes_in_sync function will give green light and
+    #     no further discovery is taken place.
+    # (4) Pod with ordinal 0 will get are_nodes_in_sync=true and cluster
+    #     setup will fail.
+
+    ordinal_of_this_pod = int(os.getenv("HOSTNAME").split("-")[-1])
+    expected_ordinals = set(range(0, ordinal_of_this_pod))
+    found_ordinals = Set()
     for name in names:
-        uri = "http://127.0.0.1:5986/_nodes/couchdb@{0}".format(name)
-        doc = {}
-        if creds[0] and creds[1]:
-            resp = requests.put(uri, data=json.dumps(doc), auth=creds)
-        else:
-            resp = requests.put(uri, data=json.dumps(doc))
-        while resp.status_code != 201 and resp.status_code != 409:
-            print('Waiting for _nodes DB to be created.',uri,'returned', resp.status_code, resp.json(), flush=True)
-            time.sleep(5)
+        # Get the podname, get the stuff after last - and convert to int
+        found_ordinals.add(int(name.split(".",1)[0].split("-")[-1]));
+
+    print("expected_ordinals",expected_ordinals)
+    print("found ordnials",found_ordinals)
+
+    # Are all expected_ordinals are part of found_ordinals?
+    if( expected_ordinals - found_ordinals):
+        print ('Expected to get at least pod(s)', expected_ordinals - found_ordinals, 'among the DNS records. Will retry.')
+    else:
+        creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
+        for name in names:
+            uri = "http://127.0.0.1:5986/_nodes/couchdb@{0}".format(name)
+            doc = {}
+
             if creds[0] and creds[1]:
                 resp = requests.put(uri, data=json.dumps(doc), auth=creds)
             else:
                 resp = requests.put(uri, data=json.dumps(doc))
-        if resp.status_code == 201:
-            print('Adding CouchDB cluster node', name, "to this pod's CouchDB. Response code:", resp.status_code ,flush=True)
+            while resp.status_code != 201 and resp.status_code != 409:
+                print('Waiting for _nodes DB to be created.',uri,'returned', resp.status_code, resp.json(), flush=True)
+                time.sleep(5)
+                if creds[0] and creds[1]:
+                    resp = requests.put(uri, data=json.dumps(doc), auth=creds)
+                else:
+                    resp = requests.put(uri, data=json.dumps(doc))
+            if resp.status_code == 201:
+                print('Adding CouchDB cluster node', name, "to this pod's CouchDB. Response code:", resp.status_code ,flush=True)
 
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972

From 3c86d2c1b9ce9069bc3de8cf999d77b89f60d0bc Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 09:29:48 +0200
Subject: [PATCH 41/50] no message

---
 mem3_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 36ea0a7..157d6b8 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -58,7 +58,7 @@ def connect_the_dots(names):
 
     ordinal_of_this_pod = int(os.getenv("HOSTNAME").split("-")[-1])
     expected_ordinals = set(range(0, ordinal_of_this_pod))
-    found_ordinals = Set()
+    found_ordinals = set()
     for name in names:
         # Get the podname, get the stuff after last - and convert to int
         found_ordinals.add(int(name.split(".",1)[0].split("-")[-1]));

From 1ba295d9cafcfa81d2e864fa6e022e3d0275860d Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 09:51:25 +0200
Subject: [PATCH 42/50] Automate _cluster_setup

---
 mem3_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 157d6b8..182e6de 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -58,7 +58,7 @@ def connect_the_dots(names):
 
     ordinal_of_this_pod = int(os.getenv("HOSTNAME").split("-")[-1])
     expected_ordinals = set(range(0, ordinal_of_this_pod))
-    found_ordinals = set()
+    found_ordinals = set([])
     for name in names:
         # Get the podname, get the stuff after last - and convert to int
         found_ordinals.add(int(name.split(".",1)[0].split("-")[-1]));

From 0ecd368f51bc773281ff4f1b0aea275544994a52 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 10:39:17 +0200
Subject: [PATCH 43/50] Automate _cluster_setup

---
 mem3_helper.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 182e6de..5e8a556 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -66,28 +66,21 @@ def connect_the_dots(names):
     print("expected_ordinals",expected_ordinals)
     print("found ordnials",found_ordinals)
 
-    # Are all expected_ordinals are part of found_ordinals?
+    # Are there expected ordinals that are not part of the found ordinals?
     if( expected_ordinals - found_ordinals):
-        print ('Expected to get at least pod(s)', expected_ordinals - found_ordinals, 'among the DNS records. Will retry.')
+        print ('Expected to get at least the following pod ordinal(s)', expected_ordinals - found_ordinals, 'among the DNS records. Will retry.')
     else:
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         for name in names:
             uri = "http://127.0.0.1:5986/_nodes/couchdb@{0}".format(name)
             doc = {}
-
+            print('Adding CouchDB cluster node', name, "to this pod's CouchDB.")
+            print ('\tRequest: GET',uri)
             if creds[0] and creds[1]:
                 resp = requests.put(uri, data=json.dumps(doc), auth=creds)
             else:
                 resp = requests.put(uri, data=json.dumps(doc))
-            while resp.status_code != 201 and resp.status_code != 409:
-                print('Waiting for _nodes DB to be created.',uri,'returned', resp.status_code, resp.json(), flush=True)
-                time.sleep(5)
-                if creds[0] and creds[1]:
-                    resp = requests.put(uri, data=json.dumps(doc), auth=creds)
-                else:
-                    resp = requests.put(uri, data=json.dumps(doc))
-            if resp.status_code == 201:
-                print('Adding CouchDB cluster node', name, "to this pod's CouchDB. Response code:", resp.status_code ,flush=True)
+            print ("\tResponse:", setup_resp.status_code, setup_resp.json(),flush=True)
 
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972

From fccddcefeb9237d7b7d5b7aeb8ac87a8b0922ea8 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 11:17:23 +0200
Subject: [PATCH 44/50] Automate _cluster_setup

---
 mem3_helper.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 5e8a556..a994db3 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -35,10 +35,19 @@ def discover_peers(service_record):
     print("\tGot the following peers' fqdm from DNS lookup:",result,flush=True)
     return result
 
+def backoff_hdlr(details):
+    print ("Backing off {wait:0.1f} seconds afters {tries} tries "
+           "calling function {target} with args {args} and kwargs "
+           "{kwargs}".format(**details))
+
+def pod_not_ready_yet(details):
+    print ('\tConnection failure. CouchDB is not responding. Will retry.')
+
 @backoff.on_exception(
     backoff.expo,
     requests.exceptions.ConnectionError,
-    max_tries=10
+    max_tries=3,
+    on_giveup=pod_not_ready_yet
 )
 def connect_the_dots(names):
 
@@ -75,7 +84,7 @@ def connect_the_dots(names):
             uri = "http://127.0.0.1:5986/_nodes/couchdb@{0}".format(name)
             doc = {}
             print('Adding CouchDB cluster node', name, "to this pod's CouchDB.")
-            print ('\tRequest: GET',uri)
+            print ('\tRequest: PUT',uri)
             if creds[0] and creds[1]:
                 resp = requests.put(uri, data=json.dumps(doc), auth=creds)
             else:

From f478e184159d3b3c0ff646500ea7712abb4da139 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 11:33:32 +0200
Subject: [PATCH 45/50] Automate _cluster_setup

---
 mem3_helper.py | 46 ++++++++++++++++++++--------------------------
 1 file changed, 20 insertions(+), 26 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index a994db3..07de198 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -32,7 +32,7 @@ def discover_peers(service_record):
     # name to form the hostname used for the Erlang node. This feels hacky
     # but not sure of a more official answer
     result = [rdata.target.to_text()[:-1] for rdata in answers]
-    print("\tGot the following peers' fqdm from DNS lookup:",result,flush=True)
+    print("\t| Got the following peers' fqdm from DNS lookup:",result,flush=True)
     return result
 
 def backoff_hdlr(details):
@@ -40,15 +40,6 @@ def backoff_hdlr(details):
            "calling function {target} with args {args} and kwargs "
            "{kwargs}".format(**details))
 
-def pod_not_ready_yet(details):
-    print ('\tConnection failure. CouchDB is not responding. Will retry.')
-
-@backoff.on_exception(
-    backoff.expo,
-    requests.exceptions.ConnectionError,
-    max_tries=3,
-    on_giveup=pod_not_ready_yet
-)
 def connect_the_dots(names):
 
     # Ordinal Index: For a StatefulSet with N replicas, each Pod in the StatefulSet
@@ -84,12 +75,15 @@ def connect_the_dots(names):
             uri = "http://127.0.0.1:5986/_nodes/couchdb@{0}".format(name)
             doc = {}
             print('Adding CouchDB cluster node', name, "to this pod's CouchDB.")
-            print ('\tRequest: PUT',uri)
-            if creds[0] and creds[1]:
-                resp = requests.put(uri, data=json.dumps(doc), auth=creds)
-            else:
-                resp = requests.put(uri, data=json.dumps(doc))
-            print ("\tResponse:", setup_resp.status_code, setup_resp.json(),flush=True)
+            print ('\t| Request: PUT',uri)
+            try:
+                if creds[0] and creds[1]:
+                    resp = requests.put(uri, data=json.dumps(doc), auth=creds)
+                else:
+                    resp = requests.put(uri, data=json.dumps(doc))
+                print ("\t| Response:", setup_resp.status_code, setup_resp.json(),flush=True)
+            except requests.exceptions.ConnectionError:
+                print ('\t| Connection failure. CouchDB not responding. Will retry.')
 
 # Compare (json) objects - order does not matter. Credits to:
 # https://stackoverflow.com/a/25851972
@@ -115,13 +109,13 @@ def finish_cluster(names):
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
         print("== Get the cluster up and running ===")
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
-        print ('\tRequest: POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"}')
-        print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+        print ('\t| Request: POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"}')
+        print ("\t|\tResponse:", setup_resp.status_code, setup_resp.json())
         if (setup_resp.status_code == 201):
-            print ("\tSweet! Just a final check for the logs...")
+            print ("\t| Sweet! Just a final check for the logs...")
             setup_resp=requests.get("http://127.0.0.1:5984/_cluster_setup",  auth=creds)
-            print ('\tRequest: GET http://127.0.0.1:5984/_cluster_setup')
-            print ("\t\tResponse:", setup_resp.status_code, setup_resp.json())
+            print ('\t| Request: GET http://127.0.0.1:5984/_cluster_setup')
+            print ("\t|\tResponse:", setup_resp.status_code, setup_resp.json())
             print("Time to relax!")
         else:
             print('Ouch! Failed the final step finalizing the cluster.')
@@ -162,7 +156,7 @@ def are_nodes_in_sync(names):
         if len(names) < 2:
             # Minimum 2 nodes to form cluster!
             not_ready = True
-            print("\tNeed at least 2 DNS records to start with. Got ",len(names))
+            print("\t| Need at least 2 DNS records to start with. Got ",len(names))
 
         # Compare local and remote _mebership data. Make sure the set
         # of nodes match. This will ensure that the remote nodes
@@ -170,22 +164,22 @@ def are_nodes_in_sync(names):
         # _cluster_setup
         if (remote_resp.status_code == 200) and (local_resp.status_code == 200):
             if ordered(local_resp.json()) == ordered(remote_resp.json()):
-                print ("\tIn sync!")
+                print ("\t| In sync!")
             else:
                 not_ready = True
                 # Rest is for logging only...
                 records_in_local_but_not_in_remote = set(local_resp.json()['cluster_nodes']) - set(remote_resp.json()['cluster_nodes'])
                 if records_in_local_but_not_in_remote:
-                    print ("\tCluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
+                    print ("\t| Cluster members in {0} not yet present in {1}: {2}".format(os.getenv("HOSTNAME"), name.split(".",1)[0], records_in_local_but_not_in_remote))
                 records_in_remote_but_not_in_local = set(remote_resp.json()['cluster_nodes']) - set(local_resp.json()['cluster_nodes'])
                 if records_in_remote_but_not_in_local:
-                    print ("\tCluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
+                    print ("\t| Cluster members in {0} not yet present in {1}: {2}".format(name.split(".",1)[0], os.getenv("HOSTNAME"), records_in_remote_but_not_in_local))
 
             # Cover the case where local pod has 1 record only
             if len(local_resp.json()['cluster_nodes']) < 2:
                 # Minimum 2 nodes to form cluster!
                 not_ready = True
-                print("\tNeed at least 2 cluster nodes in the _membership of pod",os.getenv("HOSTNAME"))
+                print("\t| Need at least 2 cluster nodes in the _membership of pod",os.getenv("HOSTNAME"))
         else:
             not_ready = True
     return not not_ready

From 4b1ef739dffa4fbe478feb1ce85ad4441599b655 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 12:19:43 +0200
Subject: [PATCH 46/50] Automate _cluster_setup

---
 mem3_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 07de198..4599265 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -81,7 +81,7 @@ def connect_the_dots(names):
                     resp = requests.put(uri, data=json.dumps(doc), auth=creds)
                 else:
                     resp = requests.put(uri, data=json.dumps(doc))
-                print ("\t| Response:", setup_resp.status_code, setup_resp.json(),flush=True)
+                print ("\t| Response:", resp.status_code, resp.json(),flush=True)
             except requests.exceptions.ConnectionError:
                 print ('\t| Connection failure. CouchDB not responding. Will retry.')
 

From 3c9554f285a6f4bfdff7a0a2981c16277f389d3e Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Sat, 25 Aug 2018 12:59:52 +0200
Subject: [PATCH 47/50] Automate _cluster_setup

---
 mem3_helper.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 4599265..074dfc9 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -63,8 +63,8 @@ def connect_the_dots(names):
         # Get the podname, get the stuff after last - and convert to int
         found_ordinals.add(int(name.split(".",1)[0].split("-")[-1]));
 
-    print("expected_ordinals",expected_ordinals)
-    print("found ordnials",found_ordinals)
+    # print("expected_ordinals",expected_ordinals)
+    # print("found ordnials",found_ordinals)
 
     # Are there expected ordinals that are not part of the found ordinals?
     if( expected_ordinals - found_ordinals):
@@ -107,7 +107,7 @@ def finish_cluster(names):
     # on the "first" pod only with this hack:
     if (os.getenv("HOSTNAME").endswith("-0")):
         creds = (os.getenv("COUCHDB_USER"), os.getenv("COUCHDB_PASSWORD"))
-        print("== Get the cluster up and running ===")
+        print("Get the cluster up and running")
         setup_resp=requests.post("http://127.0.0.1:5984/_cluster_setup", json={"action": "finish_cluster"},  auth=creds)
         print ('\t| Request: POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"}')
         print ("\t|\tResponse:", setup_resp.status_code, setup_resp.json())
@@ -120,7 +120,7 @@ def finish_cluster(names):
         else:
             print('Ouch! Failed the final step finalizing the cluster.')
     else:
-        print('This pod is intentionally skipping the POST to http://127.0.0.1:5984/_cluster_setup {"action": "finish_cluster"}')
+        print('Intentionally skipping the POST to http://127.0.0.1:5984/_cluster_setup {"action": "finish_cluster"}')
 
 
 @backoff.on_exception(
@@ -202,5 +202,5 @@ def sleep_forever():
     if (os.getenv("COUCHDB_USER") and os.getenv("COUCHDB_PASSWORD")):
         finish_cluster(peer_names)
     else:
-        print ('Skipping cluster final setup. Username and/or password not provided')
+        print ('Skipping final cluster setup. Username and/or password not provided')
     sleep_forever()

From 534562eda4e2e2a55be2d5f688a192f01deeee92 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Wed, 29 Aug 2018 18:50:43 +0200
Subject: [PATCH 48/50] Fixed typo

---
 mem3_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mem3_helper.py b/mem3_helper.py
index 074dfc9..933b5a2 100644
--- a/mem3_helper.py
+++ b/mem3_helper.py
@@ -32,7 +32,7 @@ def discover_peers(service_record):
     # name to form the hostname used for the Erlang node. This feels hacky
     # but not sure of a more official answer
     result = [rdata.target.to_text()[:-1] for rdata in answers]
-    print("\t| Got the following peers' fqdm from DNS lookup:",result,flush=True)
+    print("\t| Got the following peers' fqdn from DNS lookup:",result,flush=True)
     return result
 
 def backoff_hdlr(details):

From ad73f89a5a77d227e2c1659061a0e82a4ab269a1 Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Wed, 10 Oct 2018 14:27:35 +0200
Subject: [PATCH 49/50] Update README.md

---
 README.md | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)

diff --git a/README.md b/README.md
index a57334e..25ff5dd 100644
--- a/README.md
+++ b/README.md
@@ -11,3 +11,115 @@ cluster is automically joined up.
 If the deployment is scaled *up* after the initial creation those new Pods will
 be automatically added. Scaling *down* does not automatically remove the Pods
 from the membership database at this time.
+
+# Added functionality on top of upstream repo (kocolosk/couchdb-statefulset-assembler)
+1. Ensure that all CouchDB containers get all peers registered in _node db_
+2. Do the POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"} on pod with ordinal nr 0 (if username and password provided)
+
+The intention is that the cluster will succeed with setup even if some of the CouchDB nodes are restarted during cluster formation (e.g. due to Liveness or Readiness settings).
+
+Below are example logs from formation of a 3-node cluster.
+```
+kubectl logs -f my-release-couchdb-2 -c couchdb-statefulset-assembler       
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+	| Got the following peers' fqdm from DNS lookup: ['my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local']
+Adding CouchDB cluster node my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Adding CouchDB cluster node my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Adding CouchDB cluster node my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Probing my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Probing my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Probing my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Cluster membership populated!
+Intentionally skipping the POST to http://127.0.0.1:5984/_cluster_setup {"action": "finish_cluster"}
+```
+
+```
+kubectl logs -f my-release-couchdb-1 -c couchdb-statefulset-assembler
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+	| Got the following peers' fqdm from DNS lookup: ['my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local']
+Adding CouchDB cluster node my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Adding CouchDB cluster node my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Adding CouchDB cluster node my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Probing my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Probing my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Probing my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Cluster membership populated!
+Intentionally skipping the POST to http://127.0.0.1:5984/_cluster_setup {"action": "finish_cluster"}
+```
+
+```
+kubectl logs -f my-release-couchdb-0 -c couchdb-statefulset-assembler
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+	| Got the following peers' fqdm from DNS lookup: ['my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local']
+Adding CouchDB cluster node my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local
+	| Response: 201 {'ok': True, 'id': 'couchdb@my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local', 'rev': '1-967a00dff5e02add41819138abb3284d'}
+Adding CouchDB cluster node my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local
+	| Response: 201 {'ok': True, 'id': 'couchdb@my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local', 'rev': '1-967a00dff5e02add41819138abb3284d'}
+Adding CouchDB cluster node my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Probing my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| Cluster members in my-release-couchdb-0 not yet present in my-release-couchdb-2: {'couchdb@my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local', 'couchdb@my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local'}
+Probing my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| Cluster members in my-release-couchdb-0 not yet present in my-release-couchdb-1: {'couchdb@my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local', 'couchdb@my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local'}
+Probing my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Resolving SRV record _couchdb._tcp.my-release-couchdb.default.svc.cluster.local
+	| Got the following peers' fqdm from DNS lookup: ['my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local', 'my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local']
+Adding CouchDB cluster node my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Adding CouchDB cluster node my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Adding CouchDB cluster node my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local to this pod's CouchDB.
+	| Request: PUT http://127.0.0.1:5986/_nodes/couchdb@my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local
+	| Response: 409 {'error': 'conflict', 'reason': 'Document update conflict.'}
+Probing my-release-couchdb-2.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Probing my-release-couchdb-1.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Probing my-release-couchdb-0.my-release-couchdb.default.svc.cluster.local for cluster membership
+	| In sync!
+Cluster membership populated!
+Get the cluster up and running
+	| Request: POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"}
+	|	Response: 201 {'ok': True}
+	| Sweet! Just a final check for the logs...
+	| Request: GET http://127.0.0.1:5984/_cluster_setup
+	|	Response: 200 {'state': 'cluster_finished'}
+Time to relax!
+```

From bbc5cc436cfb9b923f11b0f4410644002026d3fd Mon Sep 17 00:00:00 2001
From: Robert Gartman <robert.gartman@avtalsbanken.se>
Date: Wed, 10 Oct 2018 14:29:03 +0200
Subject: [PATCH 50/50] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 25ff5dd..5b9ab81 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ If the deployment is scaled *up* after the initial creation those new Pods will
 be automatically added. Scaling *down* does not automatically remove the Pods
 from the membership database at this time.
 
-# Added functionality on top of upstream repo (kocolosk/couchdb-statefulset-assembler)
+### Added functionality on top of upstream repo (kocolosk/couchdb-statefulset-assembler)
 1. Ensure that all CouchDB containers get all peers registered in _node db_
 2. Do the POST http://127.0.0.1:5984/_cluster_setup , payload {"action": "finish_cluster"} on pod with ordinal nr 0 (if username and password provided)