From d51586c9754d244596e758c0d3102d11745139a4 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Mon, 1 Feb 2016 12:06:48 -0800
Subject: [PATCH 01/23] Pulling stats about shard placement in cluster

The goal is to find the common primary shards across a set of nodes.
---
 esshards/esshards.go      | 32 ++++++++++++++++++++++++++++++++
 esshards/esshards_test.go | 16 ++++++++++++++++
 estypes/estypes.go        |  2 +-
 3 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 esshards/esshards.go
 create mode 100644 esshards/esshards_test.go

diff --git a/esshards/esshards.go b/esshards/esshards.go
new file mode 100644
index 0000000..edb9cb4
--- /dev/null
+++ b/esshards/esshards.go
@@ -0,0 +1,32 @@
+package esshards
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/lytics/escp/estypes"
+)
+
+var ErrMissing = errors.New("Error GETting _search_shards endpoint")
+
+func Get(dst string) (*estypes.SearchShardsEndpoint, error) {
+	resp, err := http.Get(dst)
+	if err != nil {
+		return nil, fmt.Errorf("error contacting source Elasticsearch: %v", err)
+	}
+
+	if resp.StatusCode == 404 {
+		return nil, ErrMissing
+	}
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("non-200 status code from source Elasticsearch: %d", resp.StatusCode)
+	}
+
+	shardInfo := estypes.NewSearchShards()
+	if err := json.NewDecoder(resp.Body).Decode(&shardInfo); err != nil {
+		return nil, err
+	}
+	return shardInfo, nil
+}
diff --git a/esshards/esshards_test.go b/esshards/esshards_test.go
new file mode 100644
index 0000000..6db4f00
--- /dev/null
+++ b/esshards/esshards_test.go
@@ -0,0 +1,16 @@
+package esshards
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestGetting(t *testing.T) {
+
+	info, err := Get("http://localhost:9200/_search_shards")
+	if err != nil {
+		t.Error(err)
+	}
+	fmt.Printf("%#v\n", info)
+
+}
diff --git a/estypes/estypes.go b/estypes/estypes.go
index 81e15c1..511f843 100644
--- a/estypes/estypes.go
+++ b/estypes/estypes.go
@@ -63,7 +63,7 @@ type ShardAttributes struct {
 	State   string `json:"state"`
 	Primary bool   `json:"primary"`
 	Node    string `json:"node"`
-	//Relocating bool   `json:"relocating_node"`
+	Relocating bool   `json:"relocating_node"`
 	Shard int    `json:"shard"`
 	Index string `json:"index"`
 }

From 3c62950a0383153b14e679d09ce4f2928a42f24b Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Mon, 1 Feb 2016 23:31:23 -0800
Subject: [PATCH 02/23] ES Shard data processing functionality now with unit
 tests

Functionality to Process shard data and make it relatable to the nodes
which host them.
---
 esshards/esshards.go      | 105 +++++++++++++++++++++++
 esshards/esshards_test.go | 176 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 279 insertions(+), 2 deletions(-)

diff --git a/esshards/esshards.go b/esshards/esshards.go
index edb9cb4..fd5e66f 100644
--- a/esshards/esshards.go
+++ b/esshards/esshards.go
@@ -30,3 +30,108 @@ func Get(dst string) (*estypes.SearchShardsEndpoint, error) {
 	}
 	return shardInfo, nil
 }
+
+//Return a map of ES node IDs with built map sets for shard IDs
+func NodeIDs(endpoint estypes.SearchShardsEndpoint) map[string]map[string][]estypes.ShardAttributes {
+	nodemap := make(map[string]map[string][]estypes.ShardAttributes)
+
+	for k, _ := range endpoint.Nodes {
+		nodemap[k] = make(map[string][]estypes.ShardAttributes)
+	}
+	return nodemap
+}
+
+func NodesFromHRName(endpoint estypes.SearchShardsEndpoint, esNames map[string]struct{}) map[string]struct{} {
+	matching := make(map[string]struct{})
+	for k, v := range endpoint.Nodes {
+		if _, e := esNames[v.Name]; e {
+			matching[k] = struct{}{}
+		}
+	}
+	return matching
+}
+
+//Given a list of ShardAttributes filter and return the Primary shards
+func PrimaryShards(shards []estypes.ShardAttributes) []estypes.ShardAttributes {
+	primaries := make([]estypes.ShardAttributes, 0, 0)
+
+	for _, v := range shards {
+		if v.Primary == true {
+			primaries = append(primaries, v)
+		}
+	}
+	return primaries
+}
+
+func FlatShardAttributes(shardList []estypes.ShardInfo) []estypes.ShardAttributes {
+	shardAttrs := make([]estypes.ShardAttributes, 0, 0)
+	for _, si := range shardList {
+		for _, s := range si {
+			shardAttrs = append(shardAttrs, s)
+		}
+	}
+	return shardAttrs
+}
+
+func ProcessShardList(shardList []estypes.ShardAttributes, nodemap map[string]map[string][]estypes.ShardAttributes) map[string]map[string][]estypes.ShardAttributes {
+	primaries := PrimaryShards(shardList)
+
+	for _, p := range primaries {
+		nodemap[p.Node][p.Index] = append(nodemap[p.Node][p.Index], p)
+	}
+	return nodemap
+}
+
+// Discover the primary indexes owned by ES Nodes
+func NodeIndexSets(info estypes.SearchShardsEndpoint) map[string]map[string]struct{} {
+
+	primaryNodes := make(map[string]map[string]struct{})
+
+	nodeids := NodeIDs(info)
+	shardAttrs := FlatShardAttributes(info.Shards)
+	primeMap := ProcessShardList(shardAttrs, nodeids)
+
+	type empty struct{}
+	for pk, pv := range primeMap {
+		for _, iv := range pv {
+			for _, s := range iv {
+				//Create map for node if new
+				if _, e := primaryNodes[pk]; !e {
+					primaryNodes[pk] = make(map[string]struct{})
+				}
+				//Assign index to Node's set
+				if _, e := primaryNodes[pk][s.Index]; !e {
+					primaryNodes[pk][s.Index] = empty{}
+				}
+			}
+		}
+	}
+	return primaryNodes
+}
+
+func CommonPrimaryIndexes(info *estypes.SearchShardsEndpoint, nodeIDs map[string]struct{}) map[string]struct{} {
+	commonIndexes := make(map[string]struct{})
+	nodeSets := NodeIndexSets(*info)
+
+	for nk, nv := range nodeSets {
+		if _, exists := nodeIDs[nk]; exists {
+			if len(commonIndexes) == 0 {
+				commonIndexes = nv
+			} else {
+				commonIndexes = MatchMaps(nv, commonIndexes)
+			}
+		}
+	}
+	return commonIndexes
+}
+
+func MatchMaps(x, y map[string]struct{}) map[string]struct{} {
+	z := make(map[string]struct{})
+
+	for k, _ := range x {
+		if _, exists := y[k]; exists {
+			z[k] = struct{}{}
+		}
+	}
+	return z
+}
diff --git a/esshards/esshards_test.go b/esshards/esshards_test.go
index 6db4f00..6a3f4a4 100644
--- a/esshards/esshards_test.go
+++ b/esshards/esshards_test.go
@@ -2,15 +2,187 @@ package esshards
 
 import (
 	"fmt"
+	"os"
 	"testing"
+
+	"github.com/lytics/escp/estypes"
 )
 
-func TestGetting(t *testing.T) {
+var endpointInfo *estypes.SearchShardsEndpoint
+
+func init() {
+	endpointInfo = estypes.NewSearchShards()
+
+	na := make(map[string]estypes.NodeAttributes)
+	na["uid1"] = estypes.NodeAttributes{Name: "es1"}
+	na["uid2"] = estypes.NodeAttributes{Name: "es2"}
+	na["uid3"] = estypes.NodeAttributes{Name: "es3"}
+	na["uid4"] = estypes.NodeAttributes{Name: "es4"}
+	endpointInfo.Nodes = na
+
+	sattr0 := estypes.ShardAttributes{Primary: true, Shard: 0, Node: "uid1", Index: "index0"}
+	sattr1 := estypes.ShardAttributes{Primary: false, Shard: 0, Node: "uid2", Index: "index0"}
+	saList := make([]estypes.ShardAttributes, 0)
+	saList = append(saList, sattr0)
+	saList = append(saList, sattr1)
+
+	sinfo0 := make([]estypes.ShardInfo, 0)
+	sinfo0 = append(sinfo0, saList)
+
+	sattr0 = estypes.ShardAttributes{Primary: true, Shard: 1, Node: "uid2", Index: "index0"}
+	sattr1 = estypes.ShardAttributes{Primary: false, Shard: 1, Node: "uid3", Index: "index0"}
+	saList = make([]estypes.ShardAttributes, 0)
+	saList = append(saList, sattr0)
+	saList = append(saList, sattr1)
+	sinfo0 = append(sinfo0, saList)
+
+	sattr0 = estypes.ShardAttributes{Primary: true, Shard: 2, Node: "uid3", Index: "index0"}
+	sattr1 = estypes.ShardAttributes{Primary: false, Shard: 2, Node: "uid1", Index: "index0"}
+	saList = make([]estypes.ShardAttributes, 0)
+	saList = append(saList, sattr0)
+	saList = append(saList, sattr1)
+	sinfo0 = append(sinfo0, saList)
+
+	//Index1
+	sattr0 = estypes.ShardAttributes{Primary: true, Shard: 0, Node: "uid4", Index: "index1"}
+	sattr1 = estypes.ShardAttributes{Primary: false, Shard: 0, Node: "uid2", Index: "index1"}
+	saList = make([]estypes.ShardAttributes, 0)
+	saList = append(saList, sattr0)
+	saList = append(saList, sattr1)
+	sinfo0 = append(sinfo0, saList)
+
+	sattr0 = estypes.ShardAttributes{Primary: true, Shard: 1, Node: "uid2", Index: "index1"}
+	sattr1 = estypes.ShardAttributes{Primary: false, Shard: 1, Node: "uid4", Index: "index1"}
+	saList = make([]estypes.ShardAttributes, 0)
+	saList = append(saList, sattr0)
+	saList = append(saList, sattr1)
+	sinfo0 = append(sinfo0, saList)
+
+	sattr0 = estypes.ShardAttributes{Primary: false, Shard: 2, Node: "uid1", Index: "index1"}
+	sattr1 = estypes.ShardAttributes{Primary: true, Shard: 2, Node: "uid4", Index: "index1"}
+	saList = make([]estypes.ShardAttributes, 0)
+	saList = append(saList, sattr0)
+	saList = append(saList, sattr1)
+	sinfo0 = append(sinfo0, saList)
+	endpointInfo.Shards = sinfo0
+}
+
+func TestMatching(t *testing.T) {
+
+	x := map[string]struct{}{"hi": struct{}{}, "cat": struct{}{}}
+	y := map[string]struct{}{"cat": struct{}{}, "neh": struct{}{}}
+
+	match := MatchMaps(x, y)
+	if len(match) > 1 {
+		t.Errorf("Match size too large: %#v\n", match)
+	}
+	if _, ex := match["cat"]; !ex {
+		t.Errorf("Error matching common elements of set")
+	}
+}
+
+func TestCommonNodes(t *testing.T) {
+	nodes := make(map[string]struct{})
+	nodes["uid1"] = struct{}{}
+	nodes["uid2"] = struct{}{}
+	nodes["uid3"] = struct{}{}
+	cmi := CommonPrimaryIndexes(endpointInfo, nodes)
+
+	if _, e := cmi["index0"]; !e {
+		t.Errorf("index0 primary shards not matched between uids[1,2,3]")
+	}
+
+	nodes = make(map[string]struct{})
+	nodes["uid2"] = struct{}{}
+	nodes["uid4"] = struct{}{}
+	cmi = CommonPrimaryIndexes(endpointInfo, nodes)
 
+	if _, e := cmi["index1"]; !e {
+		t.Errorf("index1 primary shards not matched between uids[2,4]")
+	}
+}
+
+func TestNodesHRNames(t *testing.T) {
+	HRNodes := make(map[string]struct{})
+	HRNodes["es1"] = struct{}{}
+	HRNodes["es4"] = struct{}{}
+
+	ids := NodesFromHRName(*endpointInfo, HRNodes)
+	if _, e := ids["uid4"]; !e {
+		t.Error("'uid4' not returned from human readable node names")
+	}
+	if _, e := ids["uid1"]; !e {
+		t.Error("'uid1' not returned from human readable node names")
+	}
+}
+
+func TestPrimariesPerNode(t *testing.T) {
+	primaryNodes := make(map[string]map[string]struct{})
+	info := endpointInfo
+
+	nodeids := NodeIDs(*info)
+	shardAttrs := FlatShardAttributes(info.Shards)
+	primeMap := ProcessShardList(shardAttrs, nodeids)
+
+	type empty struct{}
+	for pk, pv := range primeMap {
+		for _, iv := range pv {
+			for _, s := range iv {
+				//fmt.Printf("\t\t%s; %v\n", s.Index, s.Shard)
+				if _, e := primaryNodes[pk]; !e {
+					primaryNodes[pk] = make(map[string]struct{})
+				}
+				if _, e := primaryNodes[pk][s.Index]; !e {
+					primaryNodes[pk][s.Index] = empty{}
+				}
+			}
+		}
+	}
+	if _, e := primaryNodes["uid4"]["index1"]; !e {
+		t.Error("Primary shard of index1 not on uid4")
+	}
+
+	_, uid2_i0 := primaryNodes["uid2"]["index0"]
+	_, uid2_i1 := primaryNodes["uid2"]["index1"]
+	if !uid2_i0 || !uid2_i1 {
+		t.Error("Both indexes should have primary shards on uid2")
+	}
+
+	/*
+		for k, _ := range primaryNodes {
+			fmt.Printf("%s\n", k)
+			for ki, _ := range primaryNodes[k] {
+				fmt.Printf("\t%s\n", ki)
+			}
+		}
+	*/
+
+}
+
+//Query a local ES node and display all the indexes
+// Skipped unless TESTINT=1 is set in env
+func TestFullIntegration(t *testing.T) {
+	//Skip visual data inspection if TESTINT environment variable isn't set to "1"
+	if s := os.Getenv("TESTINT"); s != "1" {
+		return
+	}
 	info, err := Get("http://localhost:9200/_search_shards")
 	if err != nil {
 		t.Error(err)
 	}
-	fmt.Printf("%#v\n", info)
 
+	nodeids := NodeIDs(*info)
+	shardAttrs := FlatShardAttributes(info.Shards)
+
+	primeMap := ProcessShardList(shardAttrs, nodeids)
+	for pk, pv := range primeMap {
+		fmt.Printf("Node: %s\n", pk)
+		for ik, iv := range pv {
+			//fmt.Printf("\tIndex: %s\n%#v\n", ik, iv)
+			fmt.Printf("\tIndex: %s\n", ik)
+			for _, s := range iv {
+				fmt.Printf("\t\t%s; %v\n", s.Index, s.Shard)
+			}
+		}
+	}
 }

From f71538357380962c01fb5d24638af745480db0fa Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Wed, 3 Feb 2016 11:56:41 -0800
Subject: [PATCH 03/23] Main to monitor execute the common primary shards
 detection

---
 cmd/esshards/main.go | 48 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)
 create mode 100644 cmd/esshards/main.go

diff --git a/cmd/esshards/main.go b/cmd/esshards/main.go
new file mode 100644
index 0000000..d25eb55
--- /dev/null
+++ b/cmd/esshards/main.go
@@ -0,0 +1,48 @@
+package main
+
+import (
+	"flag"
+	"sort"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/lytics/escp/esshards"
+)
+
+func main() {
+	var hostAddr string
+	var nodes []string
+	var nodesRaw string
+	log.SetLevel(log.InfoLevel)
+
+	flag.StringVar(&nodesRaw, "nodes", "", "Nodes to find common primary shards")
+	flag.StringVar(&hostAddr, "host", "http://localhost:9200/", "Elasticsearch query address")
+	flag.Parse()
+
+	log.Debugf("nodesRaw: %v", nodesRaw)
+	nodes = strings.Split(nodesRaw, ",")
+
+	shardAddr := hostAddr + "_search_shards"
+	info, err := esshards.Get(shardAddr)
+	if err != nil {
+		log.Fatalf("Error querying shard info from Elasticsearch:\n%#v", err)
+	}
+
+	HRnodeSet := make(map[string]struct{})
+	for _, v := range nodes {
+		HRnodeSet[v] = struct{}{}
+	}
+	nodeIDs := esshards.NodesFromHRName(*info, HRnodeSet)
+	log.Debugf("Nodes associated: %#v", nodeIDs)
+
+	cmi := esshards.CommonPrimaryIndexes(info, nodeIDs)
+	log.Infof("Indices with primary shards common to %v:\n", nodeIDs)
+	indexes := make([]string, 0)
+	for k, _ := range cmi {
+		indexes = append(indexes, k)
+	}
+	sort.Strings(indexes)
+	for _, v := range indexes {
+		log.Infof("%s", v)
+	}
+}

From 77bf6813abd0443b16b242964a44b1828e063e82 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 5 Feb 2016 10:46:46 -0800
Subject: [PATCH 04/23] Improving visibility of node names to their UIDs.

Associating the ES node IDs to the human readable and spacified name set
in the elasticsearch.yaml. Adds an extra information to output to
validate the results.
---
 esshards/esshards.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/esshards/esshards.go b/esshards/esshards.go
index fd5e66f..8206a96 100644
--- a/esshards/esshards.go
+++ b/esshards/esshards.go
@@ -41,11 +41,11 @@ func NodeIDs(endpoint estypes.SearchShardsEndpoint) map[string]map[string][]esty
 	return nodemap
 }
 
-func NodesFromHRName(endpoint estypes.SearchShardsEndpoint, esNames map[string]struct{}) map[string]struct{} {
-	matching := make(map[string]struct{})
+func NodesFromHRName(endpoint estypes.SearchShardsEndpoint, esNames map[string]struct{}) map[string]string {
+	matching := make(map[string]string)
 	for k, v := range endpoint.Nodes {
 		if _, e := esNames[v.Name]; e {
-			matching[k] = struct{}{}
+			matching[k] = v.Name
 		}
 	}
 	return matching
@@ -109,7 +109,7 @@ func NodeIndexSets(info estypes.SearchShardsEndpoint) map[string]map[string]stru
 	return primaryNodes
 }
 
-func CommonPrimaryIndexes(info *estypes.SearchShardsEndpoint, nodeIDs map[string]struct{}) map[string]struct{} {
+func CommonPrimaryIndexes(info *estypes.SearchShardsEndpoint, nodeIDs map[string]string) map[string]struct{} {
 	commonIndexes := make(map[string]struct{})
 	nodeSets := NodeIndexSets(*info)
 

From 99f6d15284d7fce98bc064fb2bb60b4aa8bcba0d Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 4 Mar 2016 15:02:41 -0800
Subject: [PATCH 05/23] Prototyping structs to unmarshall the _stats endpoint

---
 estypes/estypes.go | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/estypes/estypes.go b/estypes/estypes.go
index 511f843..89dffbe 100644
--- a/estypes/estypes.go
+++ b/estypes/estypes.go
@@ -114,3 +114,25 @@ func (is IndexSort) Less(i, j int) bool {
 	}
 	return is[i].BytesPerShard < is[j].BytesPerShard
 }
+
+/*
+Structs for the /_stats endpoint
+*/
+type Stats struct {
+	All     StatsAll                `json:"_all"`
+	Shards  StatsShards             `json:"_shards"`
+	Indices map[string]StatsIndices `json:"indices"`
+}
+
+type StatsAll struct{}
+type StatsShards struct{}
+
+type StatsIndices struct {
+	Primaries IndexPrimary `json:"primaries"`
+	//Totals    IndexTotal   `json:"total"`
+}
+
+// Index Primary Data
+type IndexPrimary struct {
+	//Store IndexStore `json:"store"`
+}

From d9dd30b9b0142f4427cdb3b6c946a9e96b50ab13 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Mon, 7 Mar 2016 17:28:01 -0800
Subject: [PATCH 06/23] Fixing test structs which were incorrectly typed

---
 esshards/esshards_test.go | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/esshards/esshards_test.go b/esshards/esshards_test.go
index 6a3f4a4..aec25e1 100644
--- a/esshards/esshards_test.go
+++ b/esshards/esshards_test.go
@@ -82,19 +82,19 @@ func TestMatching(t *testing.T) {
 }
 
 func TestCommonNodes(t *testing.T) {
-	nodes := make(map[string]struct{})
-	nodes["uid1"] = struct{}{}
-	nodes["uid2"] = struct{}{}
-	nodes["uid3"] = struct{}{}
+	nodes := make(map[string]string)
+	nodes["uid1"] = "es1"
+	nodes["uid2"] = "es2"
+	nodes["uid3"] = "es3"
 	cmi := CommonPrimaryIndexes(endpointInfo, nodes)
 
 	if _, e := cmi["index0"]; !e {
 		t.Errorf("index0 primary shards not matched between uids[1,2,3]")
 	}
 
-	nodes = make(map[string]struct{})
-	nodes["uid2"] = struct{}{}
-	nodes["uid4"] = struct{}{}
+	nodes = make(map[string]string)
+	nodes["uid2"] = "es2"
+	nodes["uid4"] = "es4"
 	cmi = CommonPrimaryIndexes(endpointInfo, nodes)
 
 	if _, e := cmi["index1"]; !e {

From 4e02e330d7760eb65ed18af5e93d366509be40fc Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 18 Mar 2016 14:41:41 -0700
Subject: [PATCH 07/23] Fixing docs

---
 cmd/esshards/main.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/esshards/main.go b/cmd/esshards/main.go
index d25eb55..bc302ac 100644
--- a/cmd/esshards/main.go
+++ b/cmd/esshards/main.go
@@ -15,7 +15,7 @@ func main() {
 	var nodesRaw string
 	log.SetLevel(log.InfoLevel)
 
-	flag.StringVar(&nodesRaw, "nodes", "", "Nodes to find common primary shards")
+	flag.StringVar(&nodesRaw, "nodes", "", "Nodes to find common primary shards eg: \"es1,es2,es3\"")
 	flag.StringVar(&hostAddr, "host", "http://localhost:9200/", "Elasticsearch query address")
 	flag.Parse()
 

From 45848d84faffbdda59b5edd7238745197e2fe1b4 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 18 Mar 2016 17:22:07 -0700
Subject: [PATCH 08/23] Quick prototype of analyzing and sort indexes on a size
 per shard basis

---
 .gitignore            |  4 +++
 cmd/shardsize/main.go | 67 +++++++++++++++++++++++++++++++++++++++++++
 esstats/esstats.go    | 27 +++++++++++++++++
 estypes/estypes.go    | 27 ++++++++++++++++-
 4 files changed, 124 insertions(+), 1 deletion(-)
 create mode 100644 cmd/shardsize/main.go
 create mode 100644 esstats/esstats.go

diff --git a/.gitignore b/.gitignore
index 72f7859..a46cdf3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,8 @@
 cmd/escp/escp
 cmd/esdiff/esdiff
+cmd/esshards/esshards
+cmd/shardsize/shardsize
 
 .*.swp
+
+
diff --git a/cmd/shardsize/main.go b/cmd/shardsize/main.go
new file mode 100644
index 0000000..c52fabe
--- /dev/null
+++ b/cmd/shardsize/main.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+	"flag"
+	"sort"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/lytics/escp/esshards"
+	"github.com/lytics/escp/esstats"
+	"github.com/lytics/escp/estypes"
+)
+
+func main() {
+	var hostAddr string
+	var nodesRaw string
+	log.SetLevel(log.InfoLevel)
+
+	flag.StringVar(&nodesRaw, "nodes", "", "Nodes to find common primary shards eg: \"es1,es2,es3\"")
+	flag.StringVar(&hostAddr, "host", "http://localhost:9200/", "Elasticsearch query address")
+	flag.Parse()
+
+	log.Debugf("nodesRaw: %v", nodesRaw)
+	//nodes = strings.Split(nodesRaw, ",")
+
+	shardAddr := hostAddr + "_stats"
+	stats, err := esstats.Get(shardAddr)
+	if err != nil {
+		log.Fatalf("Error querying shard info from Elasticsearch:\n%#v", err)
+	}
+
+	indices := make(map[string]estypes.IndexInfo)
+	for k, v := range stats.Indices {
+		indices[k] = estypes.IndexInfo{Name: k, ByteSize: v.Primaries.Store.IndexByteSize}
+	}
+
+	shardInfo, err := esshards.Get(hostAddr + "_search_shards")
+	if err != nil {
+		log.Fatalf("Error querying shard info from Elasticsearch:\n%#v", err)
+	}
+	shards := shardInfo.Shards
+	shardCount := make(map[string]int)
+	for _, s := range shards {
+		//log.Infof("%s %d", s.Index, s.Shard)
+		//log.Infof("%#v", s[0])
+		//log.Infof("%s %d %v", s[0].Index, s[0].Shard, s[0].Primary)
+		if _, ok := shardCount[s[0].Index]; !ok {
+			shardCount[s[0].Index] = 1
+		} else {
+			shardCount[s[0].Index]++
+		}
+	}
+
+	indexList := make([]estypes.IndexInfo, 0)
+	for k, v := range shardCount {
+		ii := indices[k]
+		ii.ShardCount = v
+		indices[k] = ii
+		indexList = append(indexList, ii)
+		//log.Infof("%#v", indices[k])
+	}
+
+	sort.Sort(estypes.IndexSort(indexList))
+	for i, sc := range indexList {
+		log.Infof("%d: %#v", i, sc)
+	}
+
+}
diff --git a/esstats/esstats.go b/esstats/esstats.go
new file mode 100644
index 0000000..da356f7
--- /dev/null
+++ b/esstats/esstats.go
@@ -0,0 +1,27 @@
+package esstats
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/lytics/escp/estypes"
+)
+
+func Get(qry string) (*estypes.Stats, error) {
+	resp, err := http.Get(qry)
+	if err != nil {
+		return nil, fmt.Errorf("error contacting source Elasticsearch: %v", err)
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("non-200 status code from source Elasticsearch: %d", resp.StatusCode)
+	}
+
+	statsInfo := &estypes.Stats{Indices: make(map[string]estypes.StatsIndices)}
+
+	if err := json.NewDecoder(resp.Body).Decode(&statsInfo); err != nil {
+		return nil, err
+	}
+	return statsInfo, nil
+}
diff --git a/estypes/estypes.go b/estypes/estypes.go
index 89dffbe..b1d09ab 100644
--- a/estypes/estypes.go
+++ b/estypes/estypes.go
@@ -134,5 +134,30 @@ type StatsIndices struct {
 
 // Index Primary Data
 type IndexPrimary struct {
-	//Store IndexStore `json:"store"`
+	Store IndexStore `json:"store"`
+}
+
+type IndexStore struct {
+	IndexByteSize int `json:"size_in_bytes"`
+}
+
+type IndexInfo struct {
+	Name          string
+	ByteSize      int
+	ShardCount    int
+	BytesPerShard int
+}
+
+type IndexSort []IndexInfo
+
+func (is IndexSort) Len() int      { return len(is) }
+func (is IndexSort) Swap(i, j int) { is[i], is[j] = is[j], is[i] }
+func (is IndexSort) Less(i, j int) bool {
+	if is[i].BytesPerShard == 0 {
+		is[i].BytesPerShard = is[i].ByteSize / is[i].ShardCount
+	}
+	if is[j].BytesPerShard == 0 {
+		is[j].BytesPerShard = is[j].ByteSize / is[j].ShardCount
+	}
+	return is[i].BytesPerShard < is[j].BytesPerShard
 }

From 03a30f865aa90bacb65ef11b76c7cfc56c312478 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 25 Mar 2016 16:05:42 -0700
Subject: [PATCH 09/23] Abstracting out useful functionality from the shardsize
 entry point

---
 cmd/shardsize/main.go | 15 ++-------------
 esstats/esstats.go    | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/cmd/shardsize/main.go b/cmd/shardsize/main.go
index c52fabe..3484625 100644
--- a/cmd/shardsize/main.go
+++ b/cmd/shardsize/main.go
@@ -37,18 +37,8 @@ func main() {
 	if err != nil {
 		log.Fatalf("Error querying shard info from Elasticsearch:\n%#v", err)
 	}
-	shards := shardInfo.Shards
-	shardCount := make(map[string]int)
-	for _, s := range shards {
-		//log.Infof("%s %d", s.Index, s.Shard)
-		//log.Infof("%#v", s[0])
-		//log.Infof("%s %d %v", s[0].Index, s[0].Shard, s[0].Primary)
-		if _, ok := shardCount[s[0].Index]; !ok {
-			shardCount[s[0].Index] = 1
-		} else {
-			shardCount[s[0].Index]++
-		}
-	}
+
+	shardCount := esstats.CountShards(shardInfo.Shards)
 
 	indexList := make([]estypes.IndexInfo, 0)
 	for k, v := range shardCount {
@@ -56,7 +46,6 @@ func main() {
 		ii.ShardCount = v
 		indices[k] = ii
 		indexList = append(indexList, ii)
-		//log.Infof("%#v", indices[k])
 	}
 
 	sort.Sort(estypes.IndexSort(indexList))
diff --git a/esstats/esstats.go b/esstats/esstats.go
index da356f7..c574515 100644
--- a/esstats/esstats.go
+++ b/esstats/esstats.go
@@ -25,3 +25,17 @@ func Get(qry string) (*estypes.Stats, error) {
 	}
 	return statsInfo, nil
 }
+
+//Create a map of index name to number of shards. Function iterates over
+//list of shards returned by _stats and counts them to the index name
+func CountShards(shards estypes.ShardList) map[string]int {
+	shardCount := make(map[string]int)
+	for _, s := range shards {
+		if _, ok := shardCount[s[0].Index]; !ok {
+			shardCount[s[0].Index] = 1
+		} else {
+			shardCount[s[0].Index]++
+		}
+	}
+	return shardCount
+}

From a0b7aafbc3135604058fd2ae442c5a691e3473bd Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Thu, 7 Apr 2016 11:04:09 -0700
Subject: [PATCH 10/23] relocating_node was breaking json marshalling

---
 estypes/estypes.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/estypes/estypes.go b/estypes/estypes.go
index b1d09ab..9a788d9 100644
--- a/estypes/estypes.go
+++ b/estypes/estypes.go
@@ -63,6 +63,7 @@ type ShardAttributes struct {
 	State   string `json:"state"`
 	Primary bool   `json:"primary"`
 	Node    string `json:"node"`
+<<<<<<< HEAD
 	Relocating bool   `json:"relocating_node"`
 	Shard int    `json:"shard"`
 	Index string `json:"index"`
@@ -113,6 +114,11 @@ func (is IndexSort) Less(i, j int) bool {
 		is[j].BytesPerShard = is[j].ByteSize / is[j].ShardCount
 	}
 	return is[i].BytesPerShard < is[j].BytesPerShard
+=======
+	//Relocating bool   `json:"relocating_node"`
+	Shard int    `json:"shard"`
+	Index string `json:"index"`
+>>>>>>> relocating_node was breaking json marshalling
 }
 
 /*

From c96030de33b196fccaf0464afdf78bf12012e0d6 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 13 May 2016 16:37:41 -0700
Subject: [PATCH 11/23] Calculating optimal shard capacity based on 10GB shard
 max

---
 cmd/shardsize/main.go |  9 +++++++--
 estypes/estypes.go    | 27 +++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/cmd/shardsize/main.go b/cmd/shardsize/main.go
index 3484625..f950fcc 100644
--- a/cmd/shardsize/main.go
+++ b/cmd/shardsize/main.go
@@ -8,6 +8,7 @@ import (
 	"github.com/lytics/escp/esshards"
 	"github.com/lytics/escp/esstats"
 	"github.com/lytics/escp/estypes"
+	"github.com/pivotal-golang/bytefmt"
 )
 
 func main() {
@@ -44,13 +45,17 @@ func main() {
 	for k, v := range shardCount {
 		ii := indices[k]
 		ii.ShardCount = v
+		ii.CalculateShards()
 		indices[k] = ii
 		indexList = append(indexList, ii)
 	}
 
 	sort.Sort(estypes.IndexSort(indexList))
-	for i, sc := range indexList {
-		log.Infof("%d: %#v", i, sc)
+	log.Infof("         Index          Size   Shards (Optimal Shards)")
+	for _, sc := range indexList {
+		if sc.OptimalShards-sc.ShardCount > 10 {
+			log.Infof("%20s: %7s  %3d:%8d", sc.Name, bytefmt.ByteSize(uint64(sc.ByteSize)), sc.ShardCount, sc.OptimalShards)
+		}
 	}
 
 }
diff --git a/estypes/estypes.go b/estypes/estypes.go
index 9a788d9..6cb7d85 100644
--- a/estypes/estypes.go
+++ b/estypes/estypes.go
@@ -3,6 +3,7 @@ package estypes
 import (
 	"encoding/json"
 	"errors"
+	"math"
 )
 
 type Meta struct {
@@ -100,6 +101,32 @@ type IndexInfo struct {
 	ByteSize      int
 	ShardCount    int
 	BytesPerShard int
+	OptimalShards int
+}
+
+var GBytes = 10737418240
+
+// https://github.com/golang/go/issues/4594#issuecomment-135336012
+func round(f float64) int {
+	if math.Abs(f) < 0.5 {
+		return 0
+	}
+	return int(f + math.Copysign(0.5, f))
+}
+
+func optimalShards(ii IndexInfo) int {
+	proactiveSize := float64(ii.ByteSize) * 1.25
+	proactiveShardCount := round(proactiveSize / float64(GBytes))
+
+	if proactiveShardCount <= 3 {
+		return 3
+	} else {
+		return proactiveShardCount
+	}
+}
+
+func (ii *IndexInfo) CalculateShards() {
+	ii.OptimalShards = optimalShards(*ii)
 }
 
 type IndexSort []IndexInfo

From 033d72edacc1b2b4f9447ff9ea0487805ee0a869 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 4 Nov 2016 14:00:40 -0700
Subject: [PATCH 12/23] esshards command readme and description

---
 cmd/shardsize/README.md | 12 ++++++++++++
 cmd/shardsize/main.go   |  6 ------
 2 files changed, 12 insertions(+), 6 deletions(-)
 create mode 100644 cmd/shardsize/README.md

diff --git a/cmd/shardsize/README.md b/cmd/shardsize/README.md
new file mode 100644
index 0000000..6ac6c70
--- /dev/null
+++ b/cmd/shardsize/README.md
@@ -0,0 +1,12 @@
+Shardsize
+=========
+
+Returns a list of indexes whose shards are larger than 10GB.
+
+Large shard sizes can be problematic in elasticsearch. There is no clear optimal shard size; it is mostly arbitrated by your data structures.
+
+`cmd/shardsize` calculates which indicies have excessively large shards and prints them to stdout.
+
+#### eg
+`./shardsize --host=http://localhost:9200`
+
diff --git a/cmd/shardsize/main.go b/cmd/shardsize/main.go
index f950fcc..8721671 100644
--- a/cmd/shardsize/main.go
+++ b/cmd/shardsize/main.go
@@ -13,16 +13,11 @@ import (
 
 func main() {
 	var hostAddr string
-	var nodesRaw string
 	log.SetLevel(log.InfoLevel)
 
-	flag.StringVar(&nodesRaw, "nodes", "", "Nodes to find common primary shards eg: \"es1,es2,es3\"")
 	flag.StringVar(&hostAddr, "host", "http://localhost:9200/", "Elasticsearch query address")
 	flag.Parse()
 
-	log.Debugf("nodesRaw: %v", nodesRaw)
-	//nodes = strings.Split(nodesRaw, ",")
-
 	shardAddr := hostAddr + "_stats"
 	stats, err := esstats.Get(shardAddr)
 	if err != nil {
@@ -57,5 +52,4 @@ func main() {
 			log.Infof("%20s: %7s  %3d:%8d", sc.Name, bytefmt.ByteSize(uint64(sc.ByteSize)), sc.ShardCount, sc.OptimalShards)
 		}
 	}
-
 }

From 753e26caba8ed7c88a072bfc366b1afa89a64219 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 4 Nov 2016 14:11:05 -0700
Subject: [PATCH 13/23] Renaming esshards to primeshards for clarity; adding
 documentation

---
 cmd/primeshards/README.md             | 14 ++++++++++++++
 cmd/{esshards => primeshards}/main.go |  0
 2 files changed, 14 insertions(+)
 create mode 100644 cmd/primeshards/README.md
 rename cmd/{esshards => primeshards}/main.go (100%)

diff --git a/cmd/primeshards/README.md b/cmd/primeshards/README.md
new file mode 100644
index 0000000..61d577d
--- /dev/null
+++ b/cmd/primeshards/README.md
@@ -0,0 +1,14 @@
+primeshards
+===========
+
+Correlates indexes which have primary shards on a subset of nodes. Useful for narrowing down hot indexes and their subsequent shards by host.
+
+If a single primary shard is allocated on one of the given nodes, the index is added to the returned index list. 
+
+```
+Usage of ./primeshards:
+  -host string
+    	Elasticsearch query address (default "http://localhost:9200/")
+  -nodes string
+    	Nodes to find common primary shards eg: "es1,es2,es3"
+```
diff --git a/cmd/esshards/main.go b/cmd/primeshards/main.go
similarity index 100%
rename from cmd/esshards/main.go
rename to cmd/primeshards/main.go

From b70318db615e8dad3aada42764efe460bbf0b341 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 4 Nov 2016 14:16:27 -0700
Subject: [PATCH 14/23] Ignoring primeshards binary

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index a46cdf3..907b5ef 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 cmd/escp/escp
 cmd/esdiff/esdiff
 cmd/esshards/esshards
+cmd/primeshards/primeshards
 cmd/shardsize/shardsize
 
 .*.swp

From e913ae5bdbc74b135a5575b4ad9738e124e5a2de Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Wed, 21 Dec 2016 16:31:19 -0800
Subject: [PATCH 15/23] Write the docs!

---
 esshards/esshards.go | 14 +++++++++++++-
 estypes/estypes.go   |  3 +++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/esshards/esshards.go b/esshards/esshards.go
index 8206a96..7318fb9 100644
--- a/esshards/esshards.go
+++ b/esshards/esshards.go
@@ -11,6 +11,8 @@ import (
 
 var ErrMissing = errors.New("Error GETting _search_shards endpoint")
 
+// Get returns structured data from the _search_shards endpoint.
+// This includes information about the cluster's nodes and flat listing of shards.
 func Get(dst string) (*estypes.SearchShardsEndpoint, error) {
 	resp, err := http.Get(dst)
 	if err != nil {
@@ -41,6 +43,8 @@ func NodeIDs(endpoint estypes.SearchShardsEndpoint) map[string]map[string][]esty
 	return nodemap
 }
 
+// NodesFromHRName matches a list human-readable node names to their internal node IDs and returns the
+// internal IDs.
 func NodesFromHRName(endpoint estypes.SearchShardsEndpoint, esNames map[string]struct{}) map[string]string {
 	matching := make(map[string]string)
 	for k, v := range endpoint.Nodes {
@@ -51,7 +55,7 @@ func NodesFromHRName(endpoint estypes.SearchShardsEndpoint, esNames map[string]s
 	return matching
 }
 
-//Given a list of ShardAttributes filter and return the Primary shards
+// PrimaryShards accepts a list of ShardAttributes filter and return the Primary shards
 func PrimaryShards(shards []estypes.ShardAttributes) []estypes.ShardAttributes {
 	primaries := make([]estypes.ShardAttributes, 0, 0)
 
@@ -63,6 +67,8 @@ func PrimaryShards(shards []estypes.ShardAttributes) []estypes.ShardAttributes {
 	return primaries
 }
 
+// FlatShardAttributes accepts a slice ShardInfo([]ShardAttributes), and
+// unpacks all of the contained ShardAttributes into a new list to return.
 func FlatShardAttributes(shardList []estypes.ShardInfo) []estypes.ShardAttributes {
 	shardAttrs := make([]estypes.ShardAttributes, 0, 0)
 	for _, si := range shardList {
@@ -73,6 +79,9 @@ func FlatShardAttributes(shardList []estypes.ShardInfo) []estypes.ShardAttribute
 	return shardAttrs
 }
 
+// ProcessShardList calculates a nested map[Node][Index][]ShardAttributes
+// which represents [NodeName][IndexName][]PrimaryShards and is returned.
+// Exposing which Nodes host an Index's Primary shard.
 func ProcessShardList(shardList []estypes.ShardAttributes, nodemap map[string]map[string][]estypes.ShardAttributes) map[string]map[string][]estypes.ShardAttributes {
 	primaries := PrimaryShards(shardList)
 
@@ -109,6 +118,8 @@ func NodeIndexSets(info estypes.SearchShardsEndpoint) map[string]map[string]stru
 	return primaryNodes
 }
 
+// CommonPrimaryIndexes builds on NodeIndexSets(...) to produce a set of Indexes common to
+// a set of Nodes.
 func CommonPrimaryIndexes(info *estypes.SearchShardsEndpoint, nodeIDs map[string]string) map[string]struct{} {
 	commonIndexes := make(map[string]struct{})
 	nodeSets := NodeIndexSets(*info)
@@ -125,6 +136,7 @@ func CommonPrimaryIndexes(info *estypes.SearchShardsEndpoint, nodeIDs map[string
 	return commonIndexes
 }
 
+// MatchMaps compares two map sets and returns the common keys
 func MatchMaps(x, y map[string]struct{}) map[string]struct{} {
 	z := make(map[string]struct{})
 
diff --git a/estypes/estypes.go b/estypes/estypes.go
index 6cb7d85..2f71ded 100644
--- a/estypes/estypes.go
+++ b/estypes/estypes.go
@@ -125,10 +125,13 @@ func optimalShards(ii IndexInfo) int {
 	}
 }
 
+// CalculateShards estimates an optimal number of shards given the byte size of an index.
+// There's not much science here, just based on poorly performing index backpressure.
 func (ii *IndexInfo) CalculateShards() {
 	ii.OptimalShards = optimalShards(*ii)
 }
 
+// IndexSort defines sorting indexes by their byte size.
 type IndexSort []IndexInfo
 
 func (is IndexSort) Len() int      { return len(is) }

From 1d50b90feb7ee585d179d4201ea0a4ca56535971 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Thu, 22 Dec 2016 09:51:20 -0800
Subject: [PATCH 16/23] Write the docs!

---
 esshards/doc.go      | 2 ++
 esshards/esshards.go | 3 +--
 esstats/doc.go       | 3 +++
 esstats/esstats.go   | 1 +
 estypes/doc.go       | 2 +-
 5 files changed, 8 insertions(+), 3 deletions(-)
 create mode 100644 esshards/doc.go
 create mode 100644 esstats/doc.go

diff --git a/esshards/doc.go b/esshards/doc.go
new file mode 100644
index 0000000..1f8470c
--- /dev/null
+++ b/esshards/doc.go
@@ -0,0 +1,2 @@
+// ES Shards package encapsulates functionality to understand your Elasticsearch cluster's shard allocation.
+package esshards
diff --git a/esshards/esshards.go b/esshards/esshards.go
index 7318fb9..a24b218 100644
--- a/esshards/esshards.go
+++ b/esshards/esshards.go
@@ -43,8 +43,7 @@ func NodeIDs(endpoint estypes.SearchShardsEndpoint) map[string]map[string][]esty
 	return nodemap
 }
 
-// NodesFromHRName matches a list human-readable node names to their internal node IDs and returns the
-// internal IDs.
+// NodesFromHRName matches a list human-readable node names and returns a list of InternalIDs.
 func NodesFromHRName(endpoint estypes.SearchShardsEndpoint, esNames map[string]struct{}) map[string]string {
 	matching := make(map[string]string)
 	for k, v := range endpoint.Nodes {
diff --git a/esstats/doc.go b/esstats/doc.go
new file mode 100644
index 0000000..045b460
--- /dev/null
+++ b/esstats/doc.go
@@ -0,0 +1,3 @@
+// ES Stats package API wraps up all the data return from the _stats endpoint
+// and returns it in Go data structures.
+package esstats
diff --git a/esstats/esstats.go b/esstats/esstats.go
index c574515..9fa65f4 100644
--- a/esstats/esstats.go
+++ b/esstats/esstats.go
@@ -8,6 +8,7 @@ import (
 	"github.com/lytics/escp/estypes"
 )
 
+// Get and decode the response from _stats debug endpoint.
 func Get(qry string) (*estypes.Stats, error) {
 	resp, err := http.Get(qry)
 	if err != nil {
diff --git a/estypes/doc.go b/estypes/doc.go
index 1d769ac..6896d05 100644
--- a/estypes/doc.go
+++ b/estypes/doc.go
@@ -1,3 +1,3 @@
-// This package contains common Elasticsearch data types used by multiple other
+// This package contains common Elasticsearch data types used by other
 // packages.
 package estypes

From 49a24a04e15066948fe95f5619485596344a3220 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Thu, 22 Dec 2016 10:08:27 -0800
Subject: [PATCH 17/23] Trying out godoc2md for improving documentation
 consistency with Godoc

---
 cmd/primeshards/README.md | 25 ++++++++++++++-----------
 cmd/primeshards/main.go   |  6 ++++++
 2 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/cmd/primeshards/README.md b/cmd/primeshards/README.md
index 61d577d..b832d19 100644
--- a/cmd/primeshards/README.md
+++ b/cmd/primeshards/README.md
@@ -1,14 +1,17 @@
-primeshards
-===========
 
-Correlates indexes which have primary shards on a subset of nodes. Useful for narrowing down hot indexes and their subsequent shards by host.
 
-If a single primary shard is allocated on one of the given nodes, the index is added to the returned index list. 
+> primeshards
+Correlates which indexes which have one or more primary shards on a subset of Elasticsearch nodes.
 
-```
-Usage of ./primeshards:
-  -host string
-    	Elasticsearch query address (default "http://localhost:9200/")
-  -nodes string
-    	Nodes to find common primary shards eg: "es1,es2,es3"
-```
+Useful for narrowing down hot indexes and their subsequent shards by host.
+If a single primary shard is allocated on one of the given nodes, the index is added to the index list returned.
+
+Usage: see -help
+
+
+
+
+
+
+- - -
+Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md)
diff --git a/cmd/primeshards/main.go b/cmd/primeshards/main.go
index bc302ac..6644477 100644
--- a/cmd/primeshards/main.go
+++ b/cmd/primeshards/main.go
@@ -1,3 +1,9 @@
+// Correlates which indexes which have one or more primary shards on a subset of Elasticsearch nodes.
+//
+// Useful for narrowing down hot indexes and their subsequent shards by host.
+// If a single primary shard is allocated on one of the given nodes, the index is added to the index list returned.
+//
+// Usage: see -help
 package main
 
 import (

From 992edfd7e67526864895e560ac826d2e76cc9580 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Thu, 22 Dec 2016 10:29:35 -0800
Subject: [PATCH 18/23] Updated main readme to mention other tools and made
 shardsize docs godoc friendly

---
 README.md               |  5 +++++
 cmd/shardsize/README.md | 13 ++++++++++---
 cmd/shardsize/main.go   |  8 ++++++++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 369a9ef..e578d05 100644
--- a/README.md
+++ b/README.md
@@ -27,6 +27,11 @@ esdiff -d 4 http://host1:9200/ srcindex http://host2:9200 dstindex
 esdiff -d 1 http://host1:9200/ srcindex http://host2:9200 dstindex
 ```
 
+## Introspection tools
+
+* `primeshards` Correlates which indexes which have one or more primary shards on a subset of Elasticsearch nodes.
+* `shardsize`   Returns a list of indexes whose shards are larger than 10GB. 
+
 Other Tools
 -------------------------------
 * https://github.com/taskrabbit/elasticsearch-dump
diff --git a/cmd/shardsize/README.md b/cmd/shardsize/README.md
index 6ac6c70..d6131bd 100644
--- a/cmd/shardsize/README.md
+++ b/cmd/shardsize/README.md
@@ -1,6 +1,6 @@
-Shardsize
-=========
 
+
+> shardsize
 Returns a list of indexes whose shards are larger than 10GB.
 
 Large shard sizes can be problematic in elasticsearch. There is no clear optimal shard size; it is mostly arbitrated by your data structures.
@@ -8,5 +8,12 @@ Large shard sizes can be problematic in elasticsearch. There is no clear optimal
 `cmd/shardsize` calculates which indicies have excessively large shards and prints them to stdout.
 
 #### eg
-`./shardsize --host=http://localhost:9200`
+`./shardsize --host=<a href="http://localhost:9200">http://localhost:9200</a>`
+
+
+
+
+
 
+- - -
+Generated by [godoc2md](http://godoc.org/github.com/davecheney/godoc2md)
diff --git a/cmd/shardsize/main.go b/cmd/shardsize/main.go
index 8721671..956b384 100644
--- a/cmd/shardsize/main.go
+++ b/cmd/shardsize/main.go
@@ -1,3 +1,11 @@
+// Returns a list of indexes whose shards are larger than 10GB.
+//
+// Large shard sizes can be problematic in elasticsearch. There is no clear optimal shard size; it is mostly arbitrated by your data structures.
+//
+// `cmd/shardsize` calculates which indicies have excessively large shards and prints them to stdout.
+//
+// #### eg
+// `./shardsize --host=http://localhost:9200`
 package main
 
 import (

From b5d1a0b71e949c9d5b203d3e28e6ffcab57de7a8 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Thu, 22 Dec 2016 10:35:37 -0800
Subject: [PATCH 19/23] We didn't license the repo :(

---
 LICENSE   | 202 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 README.md |   4 ++
 2 files changed, 206 insertions(+)
 create mode 100644 LICENSE

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..d955221
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,202 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2016 Lytics
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
diff --git a/README.md b/README.md
index e578d05..0711c46 100644
--- a/README.md
+++ b/README.md
@@ -32,6 +32,10 @@ esdiff -d 1 http://host1:9200/ srcindex http://host2:9200 dstindex
 * `primeshards` Correlates which indexes which have one or more primary shards on a subset of Elasticsearch nodes.
 * `shardsize`   Returns a list of indexes whose shards are larger than 10GB. 
 
+### Disclaimer
+
+This toolkit has been built against Elasticsearch 2.3.2. Your mileage may very on differing versions.
+
 Other Tools
 -------------------------------
 * https://github.com/taskrabbit/elasticsearch-dump

From c7fb4529d3830b099543f44788ad8b70c1c20ccc Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Thu, 26 Jan 2017 13:47:26 -0800
Subject: [PATCH 20/23] Improving  usage output for esshards tool

---
 cmd/shardsize/main.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/cmd/shardsize/main.go b/cmd/shardsize/main.go
index 956b384..657ed09 100644
--- a/cmd/shardsize/main.go
+++ b/cmd/shardsize/main.go
@@ -10,6 +10,8 @@ package main
 
 import (
 	"flag"
+	"fmt"
+	"os"
 	"sort"
 
 	log "github.com/Sirupsen/logrus"
@@ -23,6 +25,11 @@ func main() {
 	var hostAddr string
 	log.SetLevel(log.InfoLevel)
 
+	flag.Usage = func() {
+		fmt.Fprintf(os.Stderr, "Usage: %s -host http://localhost:9200/\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+
 	flag.StringVar(&hostAddr, "host", "http://localhost:9200/", "Elasticsearch query address")
 	flag.Parse()
 

From 35f2845725583a9fd008d31a0aef7c8c8c323594 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Thu, 26 Jan 2017 13:48:21 -0800
Subject: [PATCH 21/23] Adding vendor.json as optional reference of dependency
 revisions

---
 .gitignore         |  2 +
 vendor/vendor.json | 91 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+)
 create mode 100644 vendor/vendor.json

diff --git a/.gitignore b/.gitignore
index 907b5ef..a2bc369 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,8 @@ cmd/esshards/esshards
 cmd/primeshards/primeshards
 cmd/shardsize/shardsize
 
+vendor/**/
+
 .*.swp
 
 
diff --git a/vendor/vendor.json b/vendor/vendor.json
new file mode 100644
index 0000000..6980440
--- /dev/null
+++ b/vendor/vendor.json
@@ -0,0 +1,91 @@
+{
+	"comment": "",
+	"ignore": "test",
+	"package": [
+		{
+			"checksumSHA1": "pv31U2z7ESu51w5x38+MvIO5kC4=",
+			"path": "github.com/Sirupsen/logrus",
+			"revision": "be8c024f59696b812ed7a498c6c7c4733c58a257",
+			"revisionTime": "2016-09-20T23:40:39Z"
+		},
+		{
+			"checksumSHA1": "vJFveoepc+cD62r2X06n/VWioLg=",
+			"path": "github.com/lytics/escp/cmd/escp",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "2ECoa5rulLXFmGhDphdxbtwM69w=",
+			"path": "github.com/lytics/escp/cmd/esdiff",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "BpfnwtqtKZg21GBUCgOvhpj5Q8Y=",
+			"path": "github.com/lytics/escp/cmd/primeshards",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "07MA9V2Iruzf0mcJghBbcxLOAJo=",
+			"path": "github.com/lytics/escp/cmd/shardsize",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "DLQvabxe6xU0WAzQQTpT7CgOSXM=",
+			"path": "github.com/lytics/escp/esbulk",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "dYeS2GyRO+n9Rv6EWVwC+FkQCGg=",
+			"path": "github.com/lytics/escp/esdiff",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "2CvGraMr14qCujKqunw9QwzyBLQ=",
+			"path": "github.com/lytics/escp/esindex",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "RQq4XoCye59PxJZW2j2jEBk88NU=",
+			"path": "github.com/lytics/escp/esscroll",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "81BkWEtyLDzXXOI7EDv3D+dbooE=",
+			"path": "github.com/lytics/escp/esshards",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "/ymqbthHayfyDaofjT/OpzV22sE=",
+			"path": "github.com/lytics/escp/esstats",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "UVzCr7Aw0pORf5L2iTmX7KnJNak=",
+			"path": "github.com/lytics/escp/estypes",
+			"revision": "cf7353bbee396594535cb17988b3e8cc9df5dc9a",
+			"revisionTime": "2016-12-22T18:35:37Z"
+		},
+		{
+			"checksumSHA1": "J1oI5fm4VdotEEBN+Kci8o9LJh4=",
+			"path": "github.com/pivotal-golang/bytefmt",
+			"revision": "263ce04fc1d71bc7d50c241293488b3f54c779e5",
+			"revisionTime": "2016-01-07T07:03:02Z"
+		},
+		{
+			"checksumSHA1": "w/oQQFHCnE+S+2g5D62nfY8aDuQ=",
+			"path": "golang.org/x/sys/unix",
+			"revision": "62bee037599929a6e9146f29d10dd5208c43507d",
+			"revisionTime": "2016-06-14T22:52:37Z"
+		}
+	],
+	"rootPath": "github.com/lytics/escp"
+}

From 205cee65a813c9169da5c01116d00ac6d0739d87 Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Fri, 5 May 2017 11:18:53 -0700
Subject: [PATCH 22/23] Shardsize display update

---
 cmd/shardsize/main.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/shardsize/main.go b/cmd/shardsize/main.go
index 657ed09..54bcf36 100644
--- a/cmd/shardsize/main.go
+++ b/cmd/shardsize/main.go
@@ -61,10 +61,10 @@ func main() {
 	}
 
 	sort.Sort(estypes.IndexSort(indexList))
-	log.Infof("         Index          Size   Shards (Optimal Shards)")
+	log.Infof("         Index         IndexSize  ShardSize  Shards (Optimal Shards)")
 	for _, sc := range indexList {
 		if sc.OptimalShards-sc.ShardCount > 10 {
-			log.Infof("%20s: %7s  %3d:%8d", sc.Name, bytefmt.ByteSize(uint64(sc.ByteSize)), sc.ShardCount, sc.OptimalShards)
+			log.Infof("%20s: %7s %12s    %3d %8d", sc.Name, bytefmt.ByteSize(uint64(sc.ByteSize)), bytefmt.ByteSize(uint64(sc.BytesPerShard)), sc.ShardCount, sc.OptimalShards)
 		}
 	}
 }

From 28d04ae355ead6b58a5f1a38b844ecdcf4c85c1e Mon Sep 17 00:00:00 2001
From: Josh Roppo <joshroppo@gmail.com>
Date: Wed, 28 Jun 2017 15:25:41 -0700
Subject: [PATCH 23/23] Fixing a bad merge/meld

---
 estypes/estypes.go | 63 ++++------------------------------------------
 1 file changed, 5 insertions(+), 58 deletions(-)

diff --git a/estypes/estypes.go b/estypes/estypes.go
index 2f71ded..d8e69a9 100644
--- a/estypes/estypes.go
+++ b/estypes/estypes.go
@@ -61,13 +61,12 @@ type ShardList []ShardInfo
 type ShardInfo []ShardAttributes
 
 type ShardAttributes struct {
-	State   string `json:"state"`
-	Primary bool   `json:"primary"`
-	Node    string `json:"node"`
-<<<<<<< HEAD
+	State      string `json:"state"`
+	Primary    bool   `json:"primary"`
+	Node       string `json:"node"`
 	Relocating bool   `json:"relocating_node"`
-	Shard int    `json:"shard"`
-	Index string `json:"index"`
+	Shard      int    `json:"shard"`
+	Index      string `json:"index"`
 }
 
 /*
@@ -134,58 +133,6 @@ func (ii *IndexInfo) CalculateShards() {
 // IndexSort defines sorting indexes by their byte size.
 type IndexSort []IndexInfo
 
-func (is IndexSort) Len() int      { return len(is) }
-func (is IndexSort) Swap(i, j int) { is[i], is[j] = is[j], is[i] }
-func (is IndexSort) Less(i, j int) bool {
-	if is[i].BytesPerShard == 0 {
-		is[i].BytesPerShard = is[i].ByteSize / is[i].ShardCount
-	}
-	if is[j].BytesPerShard == 0 {
-		is[j].BytesPerShard = is[j].ByteSize / is[j].ShardCount
-	}
-	return is[i].BytesPerShard < is[j].BytesPerShard
-=======
-	//Relocating bool   `json:"relocating_node"`
-	Shard int    `json:"shard"`
-	Index string `json:"index"`
->>>>>>> relocating_node was breaking json marshalling
-}
-
-/*
-Structs for the /_stats endpoint
-*/
-type Stats struct {
-	All     StatsAll                `json:"_all"`
-	Shards  StatsShards             `json:"_shards"`
-	Indices map[string]StatsIndices `json:"indices"`
-}
-
-type StatsAll struct{}
-type StatsShards struct{}
-
-type StatsIndices struct {
-	Primaries IndexPrimary `json:"primaries"`
-	//Totals    IndexTotal   `json:"total"`
-}
-
-// Index Primary Data
-type IndexPrimary struct {
-	Store IndexStore `json:"store"`
-}
-
-type IndexStore struct {
-	IndexByteSize int `json:"size_in_bytes"`
-}
-
-type IndexInfo struct {
-	Name          string
-	ByteSize      int
-	ShardCount    int
-	BytesPerShard int
-}
-
-type IndexSort []IndexInfo
-
 func (is IndexSort) Len() int      { return len(is) }
 func (is IndexSort) Swap(i, j int) { is[i], is[j] = is[j], is[i] }
 func (is IndexSort) Less(i, j int) bool {