adding todo shortcut and functions for scape html pages

Author: 7onetella

go.mod

@@ -50,6 +50,7 @@ require (
 	github.com/tebeka/snowball v0.3.0 // indirect
 	github.com/tecbot/gorocksdb v0.0.0-20191019123150-400c56251341 // indirect
 	go.etcd.io/bbolt v1.3.3 // indirect
-	golang.org/x/net v0.0.0-20190923162816-aa69164e4478 // indirect
+	golang.org/x/net v0.0.0-20190923162816-aa69164e4478
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc
 )

go.sum

@@ -208,6 +208,8 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc h1:LMEBgNcZUqXaP7evD1PZcL6EcDVa2QOFuI+cqM3+AJM=
+gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc/go.mod h1:N8UOSI6/c2yOpa/XDz3KVUiegocTziPiqNkeNTMiG1k=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

simpleapp.go

@@ -194,6 +194,10 @@ func (app *SimpleApp) GetInputCaptureFunc() func(event *tcell.EventKey) *tcell.E
 			NewDocFlow("url", app)
 			defer app.Draw()
 			return nil
+		case tcell.KeyCtrlT:
+			NewDocFlow("todo", app)
+			defer app.Draw()
+			return nil
 		case tcell.KeyCtrlC:
 			app.Exit()
 		default:

tools.go

@@ -1,13 +1,18 @@
 package minidoc
 
 import (
+	"bytes"
 	"fmt"
+	"golang.org/x/net/html"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"os/exec"
+	"strings"
 	"syscall"
 	"time"
+
+	xmlpath "gopkg.in/xmlpath.v2"
 )
 
 func OpenFileIfNoneExist(filepath, content string) error {
@@ -161,3 +166,65 @@ func HTTPGet(url string) ([]byte, error) {
 
 	return data, err
 }
+
+// ScreenScrape hits the given URL and screen scrape  then return dom like object for searching
+func ScreenScrape(url string) (*xmlpath.Node, error) {
+
+	client := http.Client{
+		Timeout: 3 * time.Second,
+	}
+
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("failed")
+	}
+
+	pageContent, err := ioutil.ReadAll(resp.Body)
+
+	reader := strings.NewReader(string(pageContent))
+	root, err := html.Parse(reader)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var b bytes.Buffer
+	html.Render(&b, root)
+	fixedHTML := b.String()
+
+	reader = strings.NewReader(fixedHTML)
+	xmlroot, xmlerr := xmlpath.ParseHTML(reader)
+
+	if xmlerr != nil {
+		log.Fatal(xmlerr)
+	}
+
+	return xmlroot, nil
+}
+
+// SearchByXPath will walk down the node and children using xpath expression
+func SearchByXPath(context *xmlpath.Node, xpath string) []*xmlpath.Node {
+	path := xmlpath.MustCompile(xpath)
+
+	nodes := make([]*xmlpath.Node, 0, 100)
+
+	iter := path.Iter(context)
+	for iter.Next() {
+		nodes = append(nodes, iter.Node())
+	}
+
+	return nodes
+}
+
+// XPathGet xpath get by index
+func XPathGet(context *xmlpath.Node, xpath string, index int) string {
+	nodes := SearchByXPath(context, xpath)
+	if index >= len(nodes) {
+		fmt.Println("failed to get ", xpath, " index:", index)
+		return ""
+	}
+	return strings.TrimSpace(nodes[index].String())
+}