diff options
Diffstat (limited to 'fun/tvl-ebooks')
-rw-r--r-- | fun/tvl-ebooks/OWNERS | 3 | ||||
-rw-r--r-- | fun/tvl-ebooks/default.nix | 7 | ||||
-rw-r--r-- | fun/tvl-ebooks/ebook-client/main.go | 170 | ||||
-rw-r--r-- | fun/tvl-ebooks/go.mod | 8 | ||||
-rw-r--r-- | fun/tvl-ebooks/go.sum | 11 | ||||
-rw-r--r-- | fun/tvl-ebooks/irc-ingest/main.go | 99 | ||||
-rw-r--r-- | fun/tvl-ebooks/make-v6/main.go | 26 | ||||
-rw-r--r-- | fun/tvl-ebooks/mkov-engine/main.go | 246 | ||||
-rw-r--r-- | fun/tvl-ebooks/parse-logs/main.go | 174 |
9 files changed, 744 insertions, 0 deletions
diff --git a/fun/tvl-ebooks/OWNERS b/fun/tvl-ebooks/OWNERS new file mode 100644 index 000000000000..7dd8c27a574c --- /dev/null +++ b/fun/tvl-ebooks/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - ben \ No newline at end of file diff --git a/fun/tvl-ebooks/default.nix b/fun/tvl-ebooks/default.nix new file mode 100644 index 000000000000..fde6e058220e --- /dev/null +++ b/fun/tvl-ebooks/default.nix @@ -0,0 +1,7 @@ +{ pkgs, ... }: + +pkgs.buildGoModule { + name = "tvl-ebooks"; + vendorSha256 = "1p7bazh2vbhvvm559bcvfff9s4yy4q9jmklxr3sfp97inwpv6hzy"; + src = ./.; +} diff --git a/fun/tvl-ebooks/ebook-client/main.go b/fun/tvl-ebooks/ebook-client/main.go new file mode 100644 index 000000000000..7954184dd7be --- /dev/null +++ b/fun/tvl-ebooks/ebook-client/main.go @@ -0,0 +1,170 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "net" + "sync" + "time" + + "github.com/go-redis/redis" + "gopkg.in/irc.v3" +) + +var messageBeat chan bool +var firstMessage chan bool +var client *irc.Client +var safeLock sync.Mutex + +func main() { + nick := flag.String("nick", "NONE", "the ircnick you want") + from := flag.String("ip", "[::1]", "src address") + flag.Parse() + + localAddrDialier := &net.Dialer{ + LocalAddr: &net.TCPAddr{ + IP: net.ParseIP(*from), + Port: 0, + }, + } + + conn, err := tls.DialWithDialer(localAddrDialier, "tcp", "chat.freenode.net:6697", &tls.Config{}) + if err != nil { + log.Fatalln(err) + } + + messageBeat = make(chan bool) + firstMessage = make(chan bool, 10) + go ircKeepalive() + + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + go func() { + for { + time.Sleep(time.Second) + r := redisc.Ping() + if r.Err() != nil { + redisc = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + } + redisc.Set(fmt.Sprintf("alive-%s", *nick), "yes", time.Second*5) + } + }() + + if *nick == "NONE" { + log.Fatalf("You must set a nick") + } + + go func() { + <-firstMessage + for { + psub := redisc.Subscribe(fmt.Sprintf("irc-%s", *nick)) + + for { + msg, err := psub.ReceiveMessage() + if err != nil { + break + } + client.WriteMessage(&irc.Message{ + Command: "PRIVMSG", + Params: []string{ + "##tvl-ebooks", + msg.Payload, + }, + }) + } + time.Sleep(time.Second * 10) + } + + }() + + go func() { + <-firstMessage + for { + psub := redisc.Subscribe(fmt.Sprintf("raw-irc-%s", *nick)) + + for { + msg, err := psub.ReceiveMessage() + if err != nil { + break + } + im := irc.Message{} + err = json.Unmarshal([]byte(msg.Payload), &im) + if err == nil { + client.WriteMessage(&im) + } + } + time.Sleep(time.Second * 10) + } + + }() + + seenMsgBefore := false + config := irc.ClientConfig{ + Nick: *nick, + User: *nick, + Name: fmt.Sprintf("%s Ebooks", *nick), + Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) { + b, _ := json.Marshal(m) + log.Printf("%#v", string(b)) + + messageBeat <- true + + if !seenMsgBefore { + firstMessage <- true + seenMsgBefore = true + } + res := redisc.Publish("ebook", string(b)) + if res.Err() != nil { + log.Printf("Publish error! %#v", err) + } + if m.Command == "001" { + // 001 is a welcome event, so we join channels there + c.Write("JOIN ##tvl-ebooks") + } + // else if m.Command == "PRIVMSG" && c.FromChannel(m) { + // // // Create a handler on all messages. + // // c.WriteMessage(&irc.Message{ + // // Command: "PRIVMSG", + // // Params: []string{ + // // m.Params[0], + // // m.Trailing(), + // // }, + // // }) + // } + }), + } + + // Create the client + client = irc.NewClient(conn, config) + err = client.Run() + if err != nil { + log.Fatalln(err) + } +} + +func ircKeepalive() { + tt := time.NewTimer(time.Second) + lastPing := time.Now() + for { + select { + case <-tt.C: + if time.Since(lastPing) > time.Minute*5 { + log.Fatalf("It's been too long since the last IRC message, blowing up") + } + break + case <-messageBeat: + lastPing = time.Now() + } + } +} diff --git a/fun/tvl-ebooks/go.mod b/fun/tvl-ebooks/go.mod new file mode 100644 index 000000000000..154fbf8216ef --- /dev/null +++ b/fun/tvl-ebooks/go.mod @@ -0,0 +1,8 @@ +module github.com/benjojo/tvl-ebooks + +go 1.14 + +require ( + github.com/go-redis/redis v6.15.9+incompatible + gopkg.in/irc.v3 v3.1.3 +) diff --git a/fun/tvl-ebooks/go.sum b/fun/tvl-ebooks/go.sum new file mode 100644 index 000000000000..bd3ff8761342 --- /dev/null +++ b/fun/tvl-ebooks/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/irc.v3 v3.1.3 h1:yeTiJ365882L8h4AnBKYfesD92y5R5ZhGiylu9DfcPY= +gopkg.in/irc.v3 v3.1.3/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/fun/tvl-ebooks/irc-ingest/main.go b/fun/tvl-ebooks/irc-ingest/main.go new file mode 100644 index 000000000000..bbd607405cdf --- /dev/null +++ b/fun/tvl-ebooks/irc-ingest/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "log" + "os" + "time" + + "github.com/go-redis/redis" + "gopkg.in/irc.v3" +) + +var messageBeat chan bool + +func main() { + conn, err := tls.Dial("tcp", "bnc.irccloud.com:6697", nil) + if err != nil { + log.Fatalln(err) + } + + messageBeat = make(chan bool) + go ircKeepalive() + + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + go func() { + for { + time.Sleep(time.Second) + r := redisc.Ping() + if r.Err() != nil { + redisc = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + } + } + }() + + config := irc.ClientConfig{ + Nick: "Benjojo", + Pass: os.Getenv("IRCCLOUD"), + User: "Benjojo", + Name: "Ben Cox", + Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) { + b, _ := json.Marshal(m) + // log.Printf("%#v", string(b)) + + messageBeat <- true + res := redisc.Publish("irccloud", string(b)) + if res.Err() != nil { + log.Printf("Publish error! %#v", err) + } + // if m.Command == "001" { + // // 001 is a welcome event, so we join channels there + // // c.Write("JOIN #bot-test-chan") + // } else if m.Command == "PRIVMSG" && c.FromChannel(m) { + // // // Create a handler on all messages. + // // c.WriteMessage(&irc.Message{ + // // Command: "PRIVMSG", + // // Params: []string{ + // // m.Params[0], + // // m.Trailing(), + // // }, + // // }) + // } + }), + } + + // Create the client + client := irc.NewClient(conn, config) + err = client.Run() + if err != nil { + log.Fatalln(err) + } +} + +func ircKeepalive() { + tt := time.NewTimer(time.Second) + lastPing := time.Now() + for { + select { + case <-tt.C: + if time.Since(lastPing) > time.Minute*5 { + log.Fatalf("It's been too long since the last IRC message, blowing up") + } + break + case <-messageBeat: + lastPing = time.Now() + } + } + +} diff --git a/fun/tvl-ebooks/make-v6/main.go b/fun/tvl-ebooks/make-v6/main.go new file mode 100644 index 000000000000..6d4d0047bbac --- /dev/null +++ b/fun/tvl-ebooks/make-v6/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "crypto/rand" + "log" + "net" +) + +func main() { + // 2a0c:2f07:29:9999:6564:5298:8413:4652 + ip := net.ParseIP("2a0c:2f07:29::") + + rand.Read(ip[6:]) + + if ip[7] > 0xaa { + ip[4] = 0x03 + ip[5] = 0x84 + if ip[7] > 0xdd { + ip[4] = 0x08 + ip[5] = 0x64 + } + } + + log.Printf("%s\n", ip) + // +} diff --git a/fun/tvl-ebooks/mkov-engine/main.go b/fun/tvl-ebooks/mkov-engine/main.go new file mode 100644 index 000000000000..ba4863414029 --- /dev/null +++ b/fun/tvl-ebooks/mkov-engine/main.go @@ -0,0 +1,246 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "math/rand" + "strconv" + "strings" + "time" + + "github.com/go-redis/redis" +) + +type incomingIRC struct { + Command string `json:"Command"` + Host string `json:"Host"` + Name string `json:"Name"` + Params []string `json:"Params"` + User string `json:"User"` +} + +var suppressionUsernames map[string]bool +var noMkov map[string]bool + +func main() { + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + fireaway := make(chan incomingIRC, 10) + suppressionUsernames = make(map[string]bool) + + suppressionList := redisc.HGetAll("suppressionList") + suppressionListA, _ := suppressionList.Result() + + suppressionListMap, _ := stringMaptoIntMap(suppressionListA) + for v, _ := range suppressionListMap { + suppressionUsernames[v] = true + suppressionUsernames[strings.ToLower(v)] = true + } + + noMkov = make(map[string]bool) + + noMkovRedis := redisc.HGetAll("nomkov") + noMkovRedisA, _ := noMkovRedis.Result() + + noMkovMap, _ := stringMaptoIntMap(noMkovRedisA) + for v, _ := range noMkovMap { + noMkov[v] = true + noMkov[strings.ToLower(v)] = true + } + + go func() { + for { + irccloudFeed := redisc.Subscribe("irccloud") + for { + msg, err := irccloudFeed.ReceiveMessage() + if err != nil { + break + } + imsg := incomingIRC{} + err = json.Unmarshal([]byte(msg.Payload), &imsg) + if err != nil { + log.Printf("Json decoding error from irccloud feed %s", err) + continue + } + + if imsg.Command == "PRIVMSG" { + if len(imsg.Params) == 2 { + if imsg.Params[0] == "##tvl" || imsg.Params[0] == "##tvlbot" { + fireaway <- imsg + } + } + } + } + time.Sleep(time.Second) + } + }() + + for msg := range fireaway { + // Learn + learnFromMessage(msg, redisc) + msg2 := generateMesasge(msg, redisc) + + // Check if we have a active log in for that user + ttl := redisc.TTL("alive-" + msg.Name + "-eb") + ttld, err := ttl.Result() + if err == nil { + redisc.Publish("irc-"+msg.Name+"-eb", msg2) + if ttld == 0 || ttld.Seconds() == -2 { + redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2) + } + } else { + redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2) + } + } +} + +func generateMesasge(msg incomingIRC, redisc *redis.Client) string { + text := msg.Params[1] + username := strings.ToLower(msg.Name) + suppressionUsernames[username] = true + suppressionUsernames[username+":"] = true + suppressionUsernames[msg.Name] = true + suppressionUsernames[msg.Name+":"] = true + redisc.HIncrBy("suppressionList", msg.Name, 1) + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + lastWord := propwords(msg.Name, words[0], redisc) + + if noMkov[username] { + lastWord = blockoutWord(lastWord) + words[0] = blockoutWord(words[0]) + } + + lastWord = filterHighlights(lastWord) + + if lastWord == "_END_" { + if noMkov[username] { + return blockoutWord(words[0]) + } + return words[0] + } + outputMsg := words[0] + " " + lastWord + " " + + for { + lastWord = propwords(username, lastWord, redisc) + if lastWord == "" || lastWord == "_END_" { + return outputMsg + } + + if noMkov[username] { + lastWord = blockoutWord(lastWord) + } + + lastWord = filterHighlights(lastWord) + + outputMsg += lastWord + " " + if len(outputMsg) > 100 { + return outputMsg + } + } +} + +// filterHighlights: tries to prevent highlights by checking against +// a map called suppressionUsernames +func filterHighlights(in string) string { + for username := range suppressionUsernames { + if strings.Contains(in, username) { + if len(in) < 2 { + in = fmt.Sprintf("%s.%s", string(in[0]), in[1:]) + return in + } + } + } + return in +} + +func blockoutWord(in string) string { + block := "" + for i := 0; i < len(in); i++ { + block += "█" + } + return block +} + +func propwords(username string, start string, redisc *redis.Client) string { + userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start)) + userHashMap, err := userHash.Result() + if err != nil { + genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start)) + userHashMap, err = genericHash.Result() + } + + userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap) + if totalVectors == 0 { + return "" + } + targetRand := rand.Intn(totalVectors) + progresRand := 0 + + for k, v := range userIntHashMap { + progresRand += v + if targetRand > progresRand { + return k + } + } + + for k, _ := range userIntHashMap { + return k + } + + return "" +} + +func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) { + outMap = make(map[string]int) + + for k, v := range in { + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + total += int(i) + outMap[k] = int(i) + } + + return outMap, total +} + +func learnFromMessage(msg incomingIRC, redisc *redis.Client) { + text := msg.Params[1] + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + username := msg.Name + + for k, word := range words { + // HINCRBY myhash field 1 + nextWord := "_END_" + if len(words)-1 != k { + nextWord = words[k+1] + } + + if !noMkov[username] { + redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1) + } + redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1) + } +} diff --git a/fun/tvl-ebooks/parse-logs/main.go b/fun/tvl-ebooks/parse-logs/main.go new file mode 100644 index 000000000000..a2d0d20eeb9c --- /dev/null +++ b/fun/tvl-ebooks/parse-logs/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "regexp" + "strconv" + "strings" + + "github.com/go-redis/redis" +) + +type incomingIRC struct { + Command string `json:"Command"` + Host string `json:"Host"` + Name string `json:"Name"` + Params []string `json:"Params"` + User string `json:"User"` +} + +var quicklogMatch = regexp.MustCompile(`<(\w+)> (.+)`) + +func main() { + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + fireaway := make(chan incomingIRC, 10) + go func() { + f, err := os.Open("tvl.txt") + if err != nil { + log.Printf("aaa %v", err) + os.Exit(0) + } + + bio := bufio.NewReader(f) + for { + line, _, err := bio.ReadLine() + if err != nil { + break + } + + sline := string(line) + + offset := strings.Index(sline, "]") + + notime := sline[offset+1:] + + if quicklogMatch.MatchString(notime) { + bits := quicklogMatch.FindAllStringSubmatch(notime, -1) + if len(bits) != 0 { + if len(bits[0]) != 0 { + a := make([]string, 2) + a[1] = bits[0][2] + ic := incomingIRC{ + Name: bits[0][1], + Params: a, + } + log.Printf("aa %#v", ic) + + fireaway <- ic + } + } + } + + } + + }() + + for msg := range fireaway { + // Learn + learnFromMessage(msg, redisc) + // os.Exit(0) + } +} + +func generateMesasge(msg incomingIRC, redisc *redis.Client) string { + text := msg.Params[1] + username := msg.Name + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + lastWord := propwords(username, words[0], redisc) + outputMsg := words[0] + " " + lastWord + " " + + for { + lastWord = propwords(username, words[0], redisc) + if lastWord == "" || lastWord == "_END_" { + return outputMsg + } + + outputMsg += lastWord + " " + if len(outputMsg) > 100 { + return outputMsg + } + } +} + +func propwords(username string, start string, redisc *redis.Client) string { + userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start)) + userHashMap, err := userHash.Result() + if err != nil { + genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start)) + userHashMap, err = genericHash.Result() + } + + userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap) + targetRand := rand.Intn(totalVectors) + progresRand := 0 + + for k, v := range userIntHashMap { + progresRand += v + if targetRand > progresRand { + return k + } + } + + for k, _ := range userIntHashMap { + return k + } + + return "" +} + +func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) { + outMap = make(map[string]int) + + for k, v := range in { + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + total += int(i) + outMap[k] = int(i) + } + + return outMap, total +} + +func learnFromMessage(msg incomingIRC, redisc *redis.Client) { + text := msg.Params[1] + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + username := msg.Name + + for k, word := range words { + // HINCRBY myhash field 1 + nextWord := "_END_" + if len(words)-1 != k { + nextWord = words[k+1] + } + + redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1) + redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1) + } +} |