feat(ebooks): Add tvl-ebooks code r/1822

Change-Id: If519e789a91fbf427373daa383c6ae00ba5e0b6c
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2007
Tested-by: BuildkiteCI
Reviewed-by: tazjin <mail@tazj.in>

8 files changed, 687 insertions, 0 deletions

diff --git a/fun/tvl-ebooks/OWNERS b/fun/tvl-ebooks/OWNERS
new file mode 100644
index 000000000000..7dd8c27a574c
--- /dev/null
+++ b/fun/tvl-ebooks/OWNERS
@@ -0,0 +1,3 @@
+inherited: true
+owners:
+  - ben
\ No newline at end of file
diff --git a/fun/tvl-ebooks/ebook-client/main.go b/fun/tvl-ebooks/ebook-client/main.go
new file mode 100644
index 000000000000..7954184dd7be
--- /dev/null
+++ b/fun/tvl-ebooks/ebook-client/main.go
@@ -0,0 +1,170 @@
+package main
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/go-redis/redis"
+	"gopkg.in/irc.v3"
+)
+
+var messageBeat chan bool
+var firstMessage chan bool
+var client *irc.Client
+var safeLock sync.Mutex
+
+func main() {
+	nick := flag.String("nick", "NONE", "the ircnick you want")
+	from := flag.String("ip", "[::1]", "src address")
+	flag.Parse()
+
+	localAddrDialier := &net.Dialer{
+		LocalAddr: &net.TCPAddr{
+			IP:   net.ParseIP(*from),
+			Port: 0,
+		},
+	}
+
+	conn, err := tls.DialWithDialer(localAddrDialier, "tcp", "chat.freenode.net:6697", &tls.Config{})
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	messageBeat = make(chan bool)
+	firstMessage = make(chan bool, 10)
+	go ircKeepalive()
+
+	redisc := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("127.0.0.1:%d", 6379),
+		Password: "", // no password set
+		DB:       0,  // use default DB
+	})
+
+	go func() {
+		for {
+			time.Sleep(time.Second)
+			r := redisc.Ping()
+			if r.Err() != nil {
+				redisc = redis.NewClient(&redis.Options{
+					Addr:     fmt.Sprintf("127.0.0.1:%d", 6379),
+					Password: "", // no password set
+					DB:       0,  // use default DB
+				})
+			}
+			redisc.Set(fmt.Sprintf("alive-%s", *nick), "yes", time.Second*5)
+		}
+	}()
+
+	if *nick == "NONE" {
+		log.Fatalf("You must set a nick")
+	}
+
+	go func() {
+		<-firstMessage
+		for {
+			psub := redisc.Subscribe(fmt.Sprintf("irc-%s", *nick))
+
+			for {
+				msg, err := psub.ReceiveMessage()
+				if err != nil {
+					break
+				}
+				client.WriteMessage(&irc.Message{
+					Command: "PRIVMSG",
+					Params: []string{
+						"##tvl-ebooks",
+						msg.Payload,
+					},
+				})
+			}
+			time.Sleep(time.Second * 10)
+		}
+
+	}()
+
+	go func() {
+		<-firstMessage
+		for {
+			psub := redisc.Subscribe(fmt.Sprintf("raw-irc-%s", *nick))
+
+			for {
+				msg, err := psub.ReceiveMessage()
+				if err != nil {
+					break
+				}
+				im := irc.Message{}
+				err = json.Unmarshal([]byte(msg.Payload), &im)
+				if err == nil {
+					client.WriteMessage(&im)
+				}
+			}
+			time.Sleep(time.Second * 10)
+		}
+
+	}()
+
+	seenMsgBefore := false
+	config := irc.ClientConfig{
+		Nick: *nick,
+		User: *nick,
+		Name: fmt.Sprintf("%s Ebooks", *nick),
+		Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) {
+			b, _ := json.Marshal(m)
+			log.Printf("%#v", string(b))
+
+			messageBeat <- true
+
+			if !seenMsgBefore {
+				firstMessage <- true
+				seenMsgBefore = true
+			}
+			res := redisc.Publish("ebook", string(b))
+			if res.Err() != nil {
+				log.Printf("Publish error! %#v", err)
+			}
+			if m.Command == "001" {
+				// 001 is a welcome event, so we join channels there
+				c.Write("JOIN ##tvl-ebooks")
+			}
+			// else if m.Command == "PRIVMSG" && c.FromChannel(m) {
+			// 	// // Create a handler on all messages.
+			// 	// c.WriteMessage(&irc.Message{
+			// 	// 	Command: "PRIVMSG",
+			// 	// 	Params: []string{
+			// 	// 		m.Params[0],
+			// 	// 		m.Trailing(),
+			// 	// 	},
+			// 	// })
+			// }
+		}),
+	}
+
+	// Create the client
+	client = irc.NewClient(conn, config)
+	err = client.Run()
+	if err != nil {
+		log.Fatalln(err)
+	}
+}
+
+func ircKeepalive() {
+	tt := time.NewTimer(time.Second)
+	lastPing := time.Now()
+	for {
+		select {
+		case <-tt.C:
+			if time.Since(lastPing) > time.Minute*5 {
+				log.Fatalf("It's been too long since the last IRC message, blowing up")
+			}
+			break
+		case <-messageBeat:
+			lastPing = time.Now()
+		}
+	}
+}
diff --git a/fun/tvl-ebooks/go.mod b/fun/tvl-ebooks/go.mod
new file mode 100644
index 000000000000..154fbf8216ef
--- /dev/null
+++ b/fun/tvl-ebooks/go.mod
@@ -0,0 +1,8 @@
+module github.com/benjojo/tvl-ebooks
+
+go 1.14
+
+require (
+	github.com/go-redis/redis v6.15.9+incompatible
+	gopkg.in/irc.v3 v3.1.3
+)
diff --git a/fun/tvl-ebooks/go.sum b/fun/tvl-ebooks/go.sum
new file mode 100644
index 000000000000..bd3ff8761342
--- /dev/null
+++ b/fun/tvl-ebooks/go.sum
@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=
+github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/irc.v3 v3.1.3 h1:yeTiJ365882L8h4AnBKYfesD92y5R5ZhGiylu9DfcPY=
+gopkg.in/irc.v3 v3.1.3/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/fun/tvl-ebooks/irc-ingest/main.go b/fun/tvl-ebooks/irc-ingest/main.go
new file mode 100644
index 000000000000..bbd607405cdf
--- /dev/null
+++ b/fun/tvl-ebooks/irc-ingest/main.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"github.com/go-redis/redis"
+	"gopkg.in/irc.v3"
+)
+
+var messageBeat chan bool
+
+func main() {
+	conn, err := tls.Dial("tcp", "bnc.irccloud.com:6697", nil)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	messageBeat = make(chan bool)
+	go ircKeepalive()
+
+	redisc := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("127.0.0.1:%d", 6379),
+		Password: "", // no password set
+		DB:       0,  // use default DB
+	})
+
+	go func() {
+		for {
+			time.Sleep(time.Second)
+			r := redisc.Ping()
+			if r.Err() != nil {
+				redisc = redis.NewClient(&redis.Options{
+					Addr:     fmt.Sprintf("127.0.0.1:%d", 6379),
+					Password: "", // no password set
+					DB:       0,  // use default DB
+				})
+			}
+		}
+	}()
+
+	config := irc.ClientConfig{
+		Nick: "Benjojo",
+		Pass: os.Getenv("IRCCLOUD"),
+		User: "Benjojo",
+		Name: "Ben Cox",
+		Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) {
+			b, _ := json.Marshal(m)
+			// log.Printf("%#v", string(b))
+
+			messageBeat <- true
+			res := redisc.Publish("irccloud", string(b))
+			if res.Err() != nil {
+				log.Printf("Publish error! %#v", err)
+			}
+			// if m.Command == "001" {
+			// 	// 001 is a welcome event, so we join channels there
+			// 	// c.Write("JOIN #bot-test-chan")
+			// } else if m.Command == "PRIVMSG" && c.FromChannel(m) {
+			// 	// // Create a handler on all messages.
+			// 	// c.WriteMessage(&irc.Message{
+			// 	// 	Command: "PRIVMSG",
+			// 	// 	Params: []string{
+			// 	// 		m.Params[0],
+			// 	// 		m.Trailing(),
+			// 	// 	},
+			// 	// })
+			// }
+		}),
+	}
+
+	// Create the client
+	client := irc.NewClient(conn, config)
+	err = client.Run()
+	if err != nil {
+		log.Fatalln(err)
+	}
+}
+
+func ircKeepalive() {
+	tt := time.NewTimer(time.Second)
+	lastPing := time.Now()
+	for {
+		select {
+		case <-tt.C:
+			if time.Since(lastPing) > time.Minute*5 {
+				log.Fatalf("It's been too long since the last IRC message, blowing up")
+			}
+			break
+		case <-messageBeat:
+			lastPing = time.Now()
+		}
+	}
+
+}
diff --git a/fun/tvl-ebooks/make-v6/main.go b/fun/tvl-ebooks/make-v6/main.go
new file mode 100644
index 000000000000..6d4d0047bbac
--- /dev/null
+++ b/fun/tvl-ebooks/make-v6/main.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	"crypto/rand"
+	"log"
+	"net"
+)
+
+func main() {
+	// 2a0c:2f07:29:9999:6564:5298:8413:4652
+	ip := net.ParseIP("2a0c:2f07:29::")
+
+	rand.Read(ip[6:])
+
+	if ip[7] > 0xaa {
+		ip[4] = 0x03
+		ip[5] = 0x84
+		if ip[7] > 0xdd {
+			ip[4] = 0x08
+			ip[5] = 0x64
+		}
+	}
+
+	log.Printf("%s\n", ip)
+	//
+}
diff --git a/fun/tvl-ebooks/mkov-engine/main.go b/fun/tvl-ebooks/mkov-engine/main.go
new file mode 100644
index 000000000000..64742fb3d662
--- /dev/null
+++ b/fun/tvl-ebooks/mkov-engine/main.go
@@ -0,0 +1,196 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"math/rand"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-redis/redis"
+)
+
+type incomingIRC struct {
+	Command string   `json:"Command"`
+	Host    string   `json:"Host"`
+	Name    string   `json:"Name"`
+	Params  []string `json:"Params"`
+	User    string   `json:"User"`
+}
+
+var supressionUsernames map[string]bool
+
+func main() {
+	redisc := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("127.0.0.1:%d", 6379),
+		Password: "", // no password set
+		DB:       0,  // use default DB
+	})
+
+	fireaway := make(chan incomingIRC, 10)
+	supressionUsernames = make(map[string]bool)
+
+	go func() {
+		for {
+			irccloudFeed := redisc.Subscribe("irccloud")
+			for {
+				msg, err := irccloudFeed.ReceiveMessage()
+				if err != nil {
+					break
+				}
+				imsg := incomingIRC{}
+				err = json.Unmarshal([]byte(msg.Payload), &imsg)
+				if err != nil {
+					log.Printf("Json decoding error from irccloud feed %s", err)
+					continue
+				}
+
+				if imsg.Command == "PRIVMSG" {
+					if len(imsg.Params) == 2 {
+						if imsg.Params[0] == "##tvl" || imsg.Params[0] == "##tvlbot" {
+							fireaway <- imsg
+						}
+					}
+				}
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+
+	for msg := range fireaway {
+		// Learn
+		learnFromMessage(msg, redisc)
+		msg2 := generateMesasge(msg, redisc)
+
+		// Check if we have a active log in for that user
+		ttl := redisc.TTL("alive-" + msg.Name + "-eb")
+		ttld, err := ttl.Result()
+		if err == nil {
+			redisc.Publish("irc-"+msg.Name+"-eb", msg2)
+			if ttld == 0 || ttld.Seconds() == -2 {
+				redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2)
+			}
+		} else {
+			redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2)
+		}
+	}
+}
+
+func generateMesasge(msg incomingIRC, redisc *redis.Client) string {
+	text := msg.Params[1]
+	username := msg.Name
+	supressionUsernames[strings.ToLower(username)] = true
+	supressionUsernames[strings.ToLower(username)+":"] = true
+
+	text = strings.ToLower(text)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ".", "", -1)
+	text = strings.Replace(text, "!", "", -1)
+	text = strings.Replace(text, "?", "", -1)
+
+	words := strings.Split(text, " ")
+	lastWord := propwords(username, words[0], redisc)
+
+	if supressionUsernames[words[0]] {
+		if len(words[0]) < 2 {
+			words[0] = "vee"
+		}
+		words[0] = fmt.Sprintf("%s.%s", string(words[0][0]), words[0][1:])
+	}
+
+	if lastWord == "_END_" {
+		return words[0]
+	}
+	outputMsg := words[0] + " " + lastWord + " "
+
+	for {
+		lastWord = propwords(username, lastWord, redisc)
+		if lastWord == "" || lastWord == "_END_" {
+			return outputMsg
+		}
+
+		if supressionUsernames[lastWord] {
+			if len(lastWord) < 2 {
+				lastWord = "vee"
+			}
+			lastWord = fmt.Sprintf("%s.%s", string(lastWord[0]), lastWord[1:])
+		}
+
+		outputMsg += lastWord + " "
+		if len(outputMsg) > 100 {
+			return outputMsg
+		}
+	}
+}
+
+func propwords(username string, start string, redisc *redis.Client) string {
+	userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start))
+	userHashMap, err := userHash.Result()
+	if err != nil {
+		genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start))
+		userHashMap, err = genericHash.Result()
+	}
+
+	userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap)
+	if totalVectors == 0 {
+		return ""
+	}
+	targetRand := rand.Intn(totalVectors)
+	progresRand := 0
+
+	for k, v := range userIntHashMap {
+		progresRand += v
+		if targetRand > progresRand {
+			return k
+		}
+	}
+
+	for k, _ := range userIntHashMap {
+		return k
+	}
+
+	return ""
+}
+
+func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) {
+	outMap = make(map[string]int)
+
+	for k, v := range in {
+		i, err := strconv.ParseInt(v, 10, 64)
+		if err != nil {
+			continue
+		}
+		total += int(i)
+		outMap[k] = int(i)
+	}
+
+	return outMap, total
+}
+
+func learnFromMessage(msg incomingIRC, redisc *redis.Client) {
+	text := msg.Params[1]
+
+	text = strings.ToLower(text)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ".", "", -1)
+	text = strings.Replace(text, "!", "", -1)
+	text = strings.Replace(text, "?", "", -1)
+
+	words := strings.Split(text, " ")
+	username := msg.Name
+
+	for k, word := range words {
+		// HINCRBY myhash field 1
+		nextWord := "_END_"
+		if len(words)-1 != k {
+			nextWord = words[k+1]
+		}
+
+		redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1)
+		redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1)
+	}
+}
diff --git a/fun/tvl-ebooks/parse-logs/main.go b/fun/tvl-ebooks/parse-logs/main.go
new file mode 100644
index 000000000000..a2d0d20eeb9c
--- /dev/null
+++ b/fun/tvl-ebooks/parse-logs/main.go
@@ -0,0 +1,174 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/go-redis/redis"
+)
+
+type incomingIRC struct {
+	Command string   `json:"Command"`
+	Host    string   `json:"Host"`
+	Name    string   `json:"Name"`
+	Params  []string `json:"Params"`
+	User    string   `json:"User"`
+}
+
+var quicklogMatch = regexp.MustCompile(`<(\w+)> (.+)`)
+
+func main() {
+	redisc := redis.NewClient(&redis.Options{
+		Addr:     fmt.Sprintf("127.0.0.1:%d", 6379),
+		Password: "", // no password set
+		DB:       0,  // use default DB
+	})
+
+	fireaway := make(chan incomingIRC, 10)
+	go func() {
+		f, err := os.Open("tvl.txt")
+		if err != nil {
+			log.Printf("aaa %v", err)
+			os.Exit(0)
+		}
+
+		bio := bufio.NewReader(f)
+		for {
+			line, _, err := bio.ReadLine()
+			if err != nil {
+				break
+			}
+
+			sline := string(line)
+
+			offset := strings.Index(sline, "]")
+
+			notime := sline[offset+1:]
+
+			if quicklogMatch.MatchString(notime) {
+				bits := quicklogMatch.FindAllStringSubmatch(notime, -1)
+				if len(bits) != 0 {
+					if len(bits[0]) != 0 {
+						a := make([]string, 2)
+						a[1] = bits[0][2]
+						ic := incomingIRC{
+							Name:   bits[0][1],
+							Params: a,
+						}
+						log.Printf("aa %#v", ic)
+
+						fireaway <- ic
+					}
+				}
+			}
+
+		}
+
+	}()
+
+	for msg := range fireaway {
+		// Learn
+		learnFromMessage(msg, redisc)
+		// os.Exit(0)
+	}
+}
+
+func generateMesasge(msg incomingIRC, redisc *redis.Client) string {
+	text := msg.Params[1]
+	username := msg.Name
+
+	text = strings.ToLower(text)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ".", "", -1)
+	text = strings.Replace(text, "!", "", -1)
+	text = strings.Replace(text, "?", "", -1)
+
+	words := strings.Split(text, " ")
+	lastWord := propwords(username, words[0], redisc)
+	outputMsg := words[0] + " " + lastWord + " "
+
+	for {
+		lastWord = propwords(username, words[0], redisc)
+		if lastWord == "" || lastWord == "_END_" {
+			return outputMsg
+		}
+
+		outputMsg += lastWord + " "
+		if len(outputMsg) > 100 {
+			return outputMsg
+		}
+	}
+}
+
+func propwords(username string, start string, redisc *redis.Client) string {
+	userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start))
+	userHashMap, err := userHash.Result()
+	if err != nil {
+		genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start))
+		userHashMap, err = genericHash.Result()
+	}
+
+	userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap)
+	targetRand := rand.Intn(totalVectors)
+	progresRand := 0
+
+	for k, v := range userIntHashMap {
+		progresRand += v
+		if targetRand > progresRand {
+			return k
+		}
+	}
+
+	for k, _ := range userIntHashMap {
+		return k
+	}
+
+	return ""
+}
+
+func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) {
+	outMap = make(map[string]int)
+
+	for k, v := range in {
+		i, err := strconv.ParseInt(v, 10, 64)
+		if err != nil {
+			continue
+		}
+		total += int(i)
+		outMap[k] = int(i)
+	}
+
+	return outMap, total
+}
+
+func learnFromMessage(msg incomingIRC, redisc *redis.Client) {
+	text := msg.Params[1]
+
+	text = strings.ToLower(text)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ",", "", -1)
+	text = strings.Replace(text, ".", "", -1)
+	text = strings.Replace(text, "!", "", -1)
+	text = strings.Replace(text, "?", "", -1)
+
+	words := strings.Split(text, " ")
+	username := msg.Name
+
+	for k, word := range words {
+		// HINCRBY myhash field 1
+		nextWord := "_END_"
+		if len(words)-1 != k {
+			nextWord = words[k+1]
+		}
+
+		redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1)
+		redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1)
+	}
+}