From a8508373ee4bb00d866315a63dccfaa924a8a6ae Mon Sep 17 00:00:00 2001 From: Ben Cartwright-Cox Date: Mon, 28 Sep 2020 00:22:05 +0100 Subject: feat(ebooks): Add tvl-ebooks code Change-Id: If519e789a91fbf427373daa383c6ae00ba5e0b6c Reviewed-on: https://cl.tvl.fyi/c/depot/+/2007 Tested-by: BuildkiteCI Reviewed-by: tazjin --- fun/tvl-ebooks/OWNERS | 3 + fun/tvl-ebooks/ebook-client/main.go | 170 +++++++++++++++++++++++++++++++ fun/tvl-ebooks/go.mod | 8 ++ fun/tvl-ebooks/go.sum | 11 ++ fun/tvl-ebooks/irc-ingest/main.go | 99 ++++++++++++++++++ fun/tvl-ebooks/make-v6/main.go | 26 +++++ fun/tvl-ebooks/mkov-engine/main.go | 196 ++++++++++++++++++++++++++++++++++++ fun/tvl-ebooks/parse-logs/main.go | 174 ++++++++++++++++++++++++++++++++ 8 files changed, 687 insertions(+) create mode 100644 fun/tvl-ebooks/OWNERS create mode 100644 fun/tvl-ebooks/ebook-client/main.go create mode 100644 fun/tvl-ebooks/go.mod create mode 100644 fun/tvl-ebooks/go.sum create mode 100644 fun/tvl-ebooks/irc-ingest/main.go create mode 100644 fun/tvl-ebooks/make-v6/main.go create mode 100644 fun/tvl-ebooks/mkov-engine/main.go create mode 100644 fun/tvl-ebooks/parse-logs/main.go diff --git a/fun/tvl-ebooks/OWNERS b/fun/tvl-ebooks/OWNERS new file mode 100644 index 0000000000..7dd8c27a57 --- /dev/null +++ b/fun/tvl-ebooks/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - ben \ No newline at end of file diff --git a/fun/tvl-ebooks/ebook-client/main.go b/fun/tvl-ebooks/ebook-client/main.go new file mode 100644 index 0000000000..7954184dd7 --- /dev/null +++ b/fun/tvl-ebooks/ebook-client/main.go @@ -0,0 +1,170 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "net" + "sync" + "time" + + "github.com/go-redis/redis" + "gopkg.in/irc.v3" +) + +var messageBeat chan bool +var firstMessage chan bool +var client *irc.Client +var safeLock sync.Mutex + +func main() { + nick := flag.String("nick", "NONE", "the ircnick you want") + from := flag.String("ip", "[::1]", "src address") + flag.Parse() + + localAddrDialier := &net.Dialer{ + LocalAddr: &net.TCPAddr{ + IP: net.ParseIP(*from), + Port: 0, + }, + } + + conn, err := tls.DialWithDialer(localAddrDialier, "tcp", "chat.freenode.net:6697", &tls.Config{}) + if err != nil { + log.Fatalln(err) + } + + messageBeat = make(chan bool) + firstMessage = make(chan bool, 10) + go ircKeepalive() + + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + go func() { + for { + time.Sleep(time.Second) + r := redisc.Ping() + if r.Err() != nil { + redisc = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + } + redisc.Set(fmt.Sprintf("alive-%s", *nick), "yes", time.Second*5) + } + }() + + if *nick == "NONE" { + log.Fatalf("You must set a nick") + } + + go func() { + <-firstMessage + for { + psub := redisc.Subscribe(fmt.Sprintf("irc-%s", *nick)) + + for { + msg, err := psub.ReceiveMessage() + if err != nil { + break + } + client.WriteMessage(&irc.Message{ + Command: "PRIVMSG", + Params: []string{ + "##tvl-ebooks", + msg.Payload, + }, + }) + } + time.Sleep(time.Second * 10) + } + + }() + + go func() { + <-firstMessage + for { + psub := redisc.Subscribe(fmt.Sprintf("raw-irc-%s", *nick)) + + for { + msg, err := psub.ReceiveMessage() + if err != nil { + break + } + im := irc.Message{} + err = json.Unmarshal([]byte(msg.Payload), &im) + if err == nil { + client.WriteMessage(&im) + } + } + time.Sleep(time.Second * 10) + } + + }() + + seenMsgBefore := false + config := irc.ClientConfig{ + Nick: *nick, + User: *nick, + Name: fmt.Sprintf("%s Ebooks", *nick), + Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) { + b, _ := json.Marshal(m) + log.Printf("%#v", string(b)) + + messageBeat <- true + + if !seenMsgBefore { + firstMessage <- true + seenMsgBefore = true + } + res := redisc.Publish("ebook", string(b)) + if res.Err() != nil { + log.Printf("Publish error! %#v", err) + } + if m.Command == "001" { + // 001 is a welcome event, so we join channels there + c.Write("JOIN ##tvl-ebooks") + } + // else if m.Command == "PRIVMSG" && c.FromChannel(m) { + // // // Create a handler on all messages. + // // c.WriteMessage(&irc.Message{ + // // Command: "PRIVMSG", + // // Params: []string{ + // // m.Params[0], + // // m.Trailing(), + // // }, + // // }) + // } + }), + } + + // Create the client + client = irc.NewClient(conn, config) + err = client.Run() + if err != nil { + log.Fatalln(err) + } +} + +func ircKeepalive() { + tt := time.NewTimer(time.Second) + lastPing := time.Now() + for { + select { + case <-tt.C: + if time.Since(lastPing) > time.Minute*5 { + log.Fatalf("It's been too long since the last IRC message, blowing up") + } + break + case <-messageBeat: + lastPing = time.Now() + } + } +} diff --git a/fun/tvl-ebooks/go.mod b/fun/tvl-ebooks/go.mod new file mode 100644 index 0000000000..154fbf8216 --- /dev/null +++ b/fun/tvl-ebooks/go.mod @@ -0,0 +1,8 @@ +module github.com/benjojo/tvl-ebooks + +go 1.14 + +require ( + github.com/go-redis/redis v6.15.9+incompatible + gopkg.in/irc.v3 v3.1.3 +) diff --git a/fun/tvl-ebooks/go.sum b/fun/tvl-ebooks/go.sum new file mode 100644 index 0000000000..bd3ff87613 --- /dev/null +++ b/fun/tvl-ebooks/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/irc.v3 v3.1.3 h1:yeTiJ365882L8h4AnBKYfesD92y5R5ZhGiylu9DfcPY= +gopkg.in/irc.v3 v3.1.3/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/fun/tvl-ebooks/irc-ingest/main.go b/fun/tvl-ebooks/irc-ingest/main.go new file mode 100644 index 0000000000..bbd607405c --- /dev/null +++ b/fun/tvl-ebooks/irc-ingest/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "log" + "os" + "time" + + "github.com/go-redis/redis" + "gopkg.in/irc.v3" +) + +var messageBeat chan bool + +func main() { + conn, err := tls.Dial("tcp", "bnc.irccloud.com:6697", nil) + if err != nil { + log.Fatalln(err) + } + + messageBeat = make(chan bool) + go ircKeepalive() + + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + go func() { + for { + time.Sleep(time.Second) + r := redisc.Ping() + if r.Err() != nil { + redisc = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + } + } + }() + + config := irc.ClientConfig{ + Nick: "Benjojo", + Pass: os.Getenv("IRCCLOUD"), + User: "Benjojo", + Name: "Ben Cox", + Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) { + b, _ := json.Marshal(m) + // log.Printf("%#v", string(b)) + + messageBeat <- true + res := redisc.Publish("irccloud", string(b)) + if res.Err() != nil { + log.Printf("Publish error! %#v", err) + } + // if m.Command == "001" { + // // 001 is a welcome event, so we join channels there + // // c.Write("JOIN #bot-test-chan") + // } else if m.Command == "PRIVMSG" && c.FromChannel(m) { + // // // Create a handler on all messages. + // // c.WriteMessage(&irc.Message{ + // // Command: "PRIVMSG", + // // Params: []string{ + // // m.Params[0], + // // m.Trailing(), + // // }, + // // }) + // } + }), + } + + // Create the client + client := irc.NewClient(conn, config) + err = client.Run() + if err != nil { + log.Fatalln(err) + } +} + +func ircKeepalive() { + tt := time.NewTimer(time.Second) + lastPing := time.Now() + for { + select { + case <-tt.C: + if time.Since(lastPing) > time.Minute*5 { + log.Fatalf("It's been too long since the last IRC message, blowing up") + } + break + case <-messageBeat: + lastPing = time.Now() + } + } + +} diff --git a/fun/tvl-ebooks/make-v6/main.go b/fun/tvl-ebooks/make-v6/main.go new file mode 100644 index 0000000000..6d4d0047bb --- /dev/null +++ b/fun/tvl-ebooks/make-v6/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "crypto/rand" + "log" + "net" +) + +func main() { + // 2a0c:2f07:29:9999:6564:5298:8413:4652 + ip := net.ParseIP("2a0c:2f07:29::") + + rand.Read(ip[6:]) + + if ip[7] > 0xaa { + ip[4] = 0x03 + ip[5] = 0x84 + if ip[7] > 0xdd { + ip[4] = 0x08 + ip[5] = 0x64 + } + } + + log.Printf("%s\n", ip) + // +} diff --git a/fun/tvl-ebooks/mkov-engine/main.go b/fun/tvl-ebooks/mkov-engine/main.go new file mode 100644 index 0000000000..64742fb3d6 --- /dev/null +++ b/fun/tvl-ebooks/mkov-engine/main.go @@ -0,0 +1,196 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "math/rand" + "strconv" + "strings" + "time" + + "github.com/go-redis/redis" +) + +type incomingIRC struct { + Command string `json:"Command"` + Host string `json:"Host"` + Name string `json:"Name"` + Params []string `json:"Params"` + User string `json:"User"` +} + +var supressionUsernames map[string]bool + +func main() { + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + fireaway := make(chan incomingIRC, 10) + supressionUsernames = make(map[string]bool) + + go func() { + for { + irccloudFeed := redisc.Subscribe("irccloud") + for { + msg, err := irccloudFeed.ReceiveMessage() + if err != nil { + break + } + imsg := incomingIRC{} + err = json.Unmarshal([]byte(msg.Payload), &imsg) + if err != nil { + log.Printf("Json decoding error from irccloud feed %s", err) + continue + } + + if imsg.Command == "PRIVMSG" { + if len(imsg.Params) == 2 { + if imsg.Params[0] == "##tvl" || imsg.Params[0] == "##tvlbot" { + fireaway <- imsg + } + } + } + } + time.Sleep(time.Second) + } + }() + + for msg := range fireaway { + // Learn + learnFromMessage(msg, redisc) + msg2 := generateMesasge(msg, redisc) + + // Check if we have a active log in for that user + ttl := redisc.TTL("alive-" + msg.Name + "-eb") + ttld, err := ttl.Result() + if err == nil { + redisc.Publish("irc-"+msg.Name+"-eb", msg2) + if ttld == 0 || ttld.Seconds() == -2 { + redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2) + } + } else { + redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2) + } + } +} + +func generateMesasge(msg incomingIRC, redisc *redis.Client) string { + text := msg.Params[1] + username := msg.Name + supressionUsernames[strings.ToLower(username)] = true + supressionUsernames[strings.ToLower(username)+":"] = true + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + lastWord := propwords(username, words[0], redisc) + + if supressionUsernames[words[0]] { + if len(words[0]) < 2 { + words[0] = "vee" + } + words[0] = fmt.Sprintf("%s.%s", string(words[0][0]), words[0][1:]) + } + + if lastWord == "_END_" { + return words[0] + } + outputMsg := words[0] + " " + lastWord + " " + + for { + lastWord = propwords(username, lastWord, redisc) + if lastWord == "" || lastWord == "_END_" { + return outputMsg + } + + if supressionUsernames[lastWord] { + if len(lastWord) < 2 { + lastWord = "vee" + } + lastWord = fmt.Sprintf("%s.%s", string(lastWord[0]), lastWord[1:]) + } + + outputMsg += lastWord + " " + if len(outputMsg) > 100 { + return outputMsg + } + } +} + +func propwords(username string, start string, redisc *redis.Client) string { + userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start)) + userHashMap, err := userHash.Result() + if err != nil { + genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start)) + userHashMap, err = genericHash.Result() + } + + userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap) + if totalVectors == 0 { + return "" + } + targetRand := rand.Intn(totalVectors) + progresRand := 0 + + for k, v := range userIntHashMap { + progresRand += v + if targetRand > progresRand { + return k + } + } + + for k, _ := range userIntHashMap { + return k + } + + return "" +} + +func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) { + outMap = make(map[string]int) + + for k, v := range in { + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + total += int(i) + outMap[k] = int(i) + } + + return outMap, total +} + +func learnFromMessage(msg incomingIRC, redisc *redis.Client) { + text := msg.Params[1] + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + username := msg.Name + + for k, word := range words { + // HINCRBY myhash field 1 + nextWord := "_END_" + if len(words)-1 != k { + nextWord = words[k+1] + } + + redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1) + redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1) + } +} diff --git a/fun/tvl-ebooks/parse-logs/main.go b/fun/tvl-ebooks/parse-logs/main.go new file mode 100644 index 0000000000..a2d0d20eeb --- /dev/null +++ b/fun/tvl-ebooks/parse-logs/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "regexp" + "strconv" + "strings" + + "github.com/go-redis/redis" +) + +type incomingIRC struct { + Command string `json:"Command"` + Host string `json:"Host"` + Name string `json:"Name"` + Params []string `json:"Params"` + User string `json:"User"` +} + +var quicklogMatch = regexp.MustCompile(`<(\w+)> (.+)`) + +func main() { + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + fireaway := make(chan incomingIRC, 10) + go func() { + f, err := os.Open("tvl.txt") + if err != nil { + log.Printf("aaa %v", err) + os.Exit(0) + } + + bio := bufio.NewReader(f) + for { + line, _, err := bio.ReadLine() + if err != nil { + break + } + + sline := string(line) + + offset := strings.Index(sline, "]") + + notime := sline[offset+1:] + + if quicklogMatch.MatchString(notime) { + bits := quicklogMatch.FindAllStringSubmatch(notime, -1) + if len(bits) != 0 { + if len(bits[0]) != 0 { + a := make([]string, 2) + a[1] = bits[0][2] + ic := incomingIRC{ + Name: bits[0][1], + Params: a, + } + log.Printf("aa %#v", ic) + + fireaway <- ic + } + } + } + + } + + }() + + for msg := range fireaway { + // Learn + learnFromMessage(msg, redisc) + // os.Exit(0) + } +} + +func generateMesasge(msg incomingIRC, redisc *redis.Client) string { + text := msg.Params[1] + username := msg.Name + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + lastWord := propwords(username, words[0], redisc) + outputMsg := words[0] + " " + lastWord + " " + + for { + lastWord = propwords(username, words[0], redisc) + if lastWord == "" || lastWord == "_END_" { + return outputMsg + } + + outputMsg += lastWord + " " + if len(outputMsg) > 100 { + return outputMsg + } + } +} + +func propwords(username string, start string, redisc *redis.Client) string { + userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start)) + userHashMap, err := userHash.Result() + if err != nil { + genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start)) + userHashMap, err = genericHash.Result() + } + + userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap) + targetRand := rand.Intn(totalVectors) + progresRand := 0 + + for k, v := range userIntHashMap { + progresRand += v + if targetRand > progresRand { + return k + } + } + + for k, _ := range userIntHashMap { + return k + } + + return "" +} + +func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) { + outMap = make(map[string]int) + + for k, v := range in { + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + total += int(i) + outMap[k] = int(i) + } + + return outMap, total +} + +func learnFromMessage(msg incomingIRC, redisc *redis.Client) { + text := msg.Params[1] + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + username := msg.Name + + for k, word := range words { + // HINCRBY myhash field 1 + nextWord := "_END_" + if len(words)-1 != k { + nextWord = words[k+1] + } + + redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1) + redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1) + } +} -- cgit 1.4.1