package main import ( "bytes" "encoding/json" "fmt" "github.com/gorilla/websocket" "index/suffixarray" "io" "io/ioutil" "log" "math/rand" "net/http" "os" "regexp" "strconv" "strings" "time" "unicode" "unicode/utf8" ) func main() { searcher := Searcher{} err := searcher.Load("completeworks.txt") if err != nil { log.Fatal(err) } searcher.completeSlice() fs := http.FileServer(http.Dir("./static")) http.Handle("/", fs) http.HandleFunc("/search", handleSearch(searcher)) http.HandleFunc("/dispatcher", searcher.dispatcher) port := os.Getenv("PORT") if port == "" { port = "3001" } fmt.Printf("Listening on port %s...\n", port) err = http.ListenAndServe(fmt.Sprintf(":%s", port), nil) if err != nil { log.Fatal(err) } } func (s *Searcher) dispatcher(w http.ResponseWriter, r *http.Request) { var upgrader = websocket.Upgrader{ReadBufferSize: 1024, WriteBufferSize: 1024, CheckOrigin: func(r *http.Request) bool { return true }, } ws, err := upgrader.Upgrade(w, r, nil) if err != nil { log.Printf("ws upgrader error: %v", err) } defer func() { err := ws.Close() if err != nil { log.Printf("ws.Close() error: %v", err) } }() cache := make(map[int]bool); colors := s.colors() for { var msg message // scroll movement err := ws.ReadJSON(&msg) if err != nil { log.Printf("read msg error: %v", err) break } // check msg.cached if len(msg.Cached) > 0 { for _, n := range msg.Cached { cache[n] = true } } if msg.Height == 0 { msg.Height = 7680 } line := msg.Requested/30; view := msg.Height/30 bacon := 0; burger := 0 for bbq, salad := range colors.lr { if bbq != 0 { bacon = colors.lr[(bbq-1)] } if line >= bacon && line <= salad { // TODO: redo, check // match burger = bbq break } } for i := 0; i < view; i++ { // total lines in view if !cache[line+i] { html := "" + webified_complete_works[line+i] + "" web := webified_complete_works[line+i] for nm, clrcd := range colors.cr[burger] { if strings.Contains(web, nm) { // check and replace coffee := make([]rune, 0); cup := "" for blen, starbucks := range web { if unicode.IsUpper(starbucks) { coffee = append(coffee, starbucks) } if unicode.IsSpace(starbucks) { coffee = append(coffee, starbucks) } if starbucks == 46 { // period, check what you have (line start, normally) cup = string(coffee); cup = strings.TrimSpace(cup) if cup == nm { // match span := ""+nm+"" html = strings.ReplaceAll(html, nm, span) } cup = "" coffee = nil coffee = make([]rune, 0) } if unicode.IsLower(starbucks) { // check what you have (mid line) cup = string(coffee); cup = strings.TrimSpace(cup) if cup == nm { // match span := ""+nm+"" html = strings.ReplaceAll(html, nm, span) } cup = "" coffee = nil coffee = make([]rune, 0) } if blen == len(web) { // reach end, check what you have cup = string(coffee); cup = strings.TrimSpace(cup) if cup == nm { // match span := ""+nm+"" html = strings.ReplaceAll(html, nm, span) } cup = "" coffee = nil coffee = make([]rune, 0) } } } } err = ws.WriteJSON(html) if err != nil { log.Printf("WriteJSON err: %v", err) } else { cache[line+i] = true } } } } } func (s *Searcher) SpeakerTwo(runes []rune, idx []int) (bool, []int) { // 136214 // TODO: MICHAEL, &c. 53556 index newline := false previous_rune, rune_pos := s.PreviousRune(idx[0]) if previous_rune == 10 { newline = true } for !unicode.IsLetter(previous_rune) { previous_rune, rune_pos = s.PreviousRune(rune_pos) if previous_rune == 10 { newline = true } } if newline == false { _, previous_word := s.PreviousWord(idx[0]) if len(previous_word) > 3 { if isUpper(string(previous_word)) { return false, nil } } } //////////////////////////////////////////////////////////////////////////////////// if len(runes) < 3 { return false, nil } for _, r := range runes { if !unicode.IsUpper(r) && unicode.IsLetter(r) { return false, nil } if unicode.IsSpace(r) { return false, nil } } // multiple word speakers go brrr next, _ := s.NextRune(idx[1]) if next == 46 { // Period, dot or full stop return true, idx } next_word_pos, next_word := s.NextWord(idx[1]) if len(next_word) > 3 { for _, r := range next_word { if !unicode.IsUpper(r) && unicode.IsLetter(r) { return false, nil } if unicode.IsSpace(r) { return false, nil } } mc, donalds := s.PreviousRune(next_word_pos[0]); countme := 0 for unicode.IsSpace(mc) { mc, donalds = s.PreviousRune(donalds) countme++ } } next, _ = s.NextRune(next_word_pos[1]) if next == 46 { // Period, dot or full stop return true, []int{idx[0], next_word_pos[1]} } return false, nil } func (s *Searcher) lineRangesToEach() []int { z := 0; nl := 0; lr := make([]int, 0) for c, r := range s.CompleteWorks { if z > len(BYTE_RANGE)-1 { break } // inc z when >= if c >= BYTE_RANGE[z][1] { z++ lr = append(lr, nl) } // count newlines if r == 10 { nl++ } } return lr } type colors struct { lr []int cr [] map[string]string } var palette = []string{"FF99C8", "FEC8C3", "FCF6BD", "E6F5CE", "D0F4DE", "BDE9EC", "A9DEF9", "C7D0F9", "E4C1F9"} func (s *Searcher) colors() colors { // and players colors := colors{ lr: s.lineRangesToEach(), } spkr := make(map[string]string, 0); rand.Seed(time.Now().Unix()) start := BYTE_RANGE[0][0] idx, word := s.NextWord(start) for md := range BYTE_RANGE { for idx[1] < BYTE_RANGE[md][1] { idx, word = s.NextWord(idx[1]) for !s.Speaker(word, idx) { idx, word = s.NextWord(idx[1]) } _, zdx := s.SpeakerTwo(word, idx) if !strings.Contains(s.CompleteWorks[zdx[0]:zdx[1]], "CORIOLANUS]") { spkr[s.CompleteWorks[zdx[0]:zdx[1]]] = palette[rand.Intn(len(palette))] } } colors.cr = append(colors.cr, spkr) spkr = nil spkr = make(map[string]string, 0) } return colors } var webified_complete_works []string func (s *Searcher) completeSlice() { bucket := make([]rune, 0) for _, r := range s.CompleteWorks { bucket = append(bucket, r) if r == 10 { webified_complete_works = append(webified_complete_works, string(bucket)) bucket = nil } } fmt.Println(len(webified_complete_works), webified_complete_works[169431]) } type message struct { Requested int `json:"requested"` Cached []int `json:"cached"` Height int `json:"height"` } type Searcher struct { CompleteWorks string SuffixArray *suffixarray.Index } func handleSearch(searcher Searcher) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { query, ok := r.URL.Query()["q"] if !ok || len(query[0]) < 1 { w.WriteHeader(http.StatusBadRequest) w.Write([]byte("missing search query in URL params")) return } results := searcher.Search(query[0]) buf := &bytes.Buffer{} enc := json.NewEncoder(buf) err := enc.Encode(results) if err != nil { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("encoding failure")) return } w.Header().Set("Content-Type", "application/json") w.Write(buf.Bytes()) } } func (s *Searcher) Load(filename string) error { dat, err := ioutil.ReadFile(filename) if err != nil { return fmt.Errorf("Load: %w", err) } s.CompleteWorks = string(dat) s.SuffixArray = suffixarray.New(dat) return nil } func (s *Searcher) runeCounter(pos int) int { rc := 0 for i, _ := range s.CompleteWorks { rc++ if i >= pos { break } } return rc } func (s *Searcher) PreviousRune(idx int) (r rune, pos int) { bucket := make([]byte, 0) idx-- if utf8.RuneStart(s.CompleteWorks[idx]) { bucket = append(bucket, s.CompleteWorks[idx]) r, _ := utf8.DecodeRune(bucket) return r, idx } else { pos := 0 for !utf8.RuneStart(s.CompleteWorks[idx]) { idx-- } pos = idx bucket = append(bucket, s.CompleteWorks[idx]) idx++ for !utf8.RuneStart(s.CompleteWorks[idx]) { bucket = append(bucket, s.CompleteWorks[idx]) idx++ } r, _ := utf8.DecodeRune(bucket) return r, pos } } func (s *Searcher) NextRune(pos int) (rune, int) { bucket := make([]byte, 0) bucket = append(bucket, s.CompleteWorks[pos]) pos++ for !utf8.RuneStart(s.CompleteWorks[pos]) { bucket = append(bucket, s.CompleteWorks[pos]) pos++ } r, _ := utf8.DecodeRune(bucket) return r, pos } func (s *Searcher) Speaker(runes []rune, idx []int) bool { // 136214 // TODO: MICHAEL, &c. 53556 index newline := false previous_rune, rune_pos := s.PreviousRune(idx[0]) if previous_rune == 10 { newline = true } for !unicode.IsLetter(previous_rune) { previous_rune, rune_pos = s.PreviousRune(rune_pos) if previous_rune == 10 { newline = true } } if newline == false { _, previous_word := s.PreviousWord(idx[0]) if len(previous_word) > 3 { if isUpper(string(previous_word)) { return false } } } if len(runes) < 3 { return false } for _, r := range runes { if !unicode.IsUpper(r) && unicode.IsLetter(r) { return false } if unicode.IsSpace(r) { return false } } // multiple word speakers go brrr next, _ := s.NextRune(idx[1]) if next == 46 { // Period, dot or full stop return true } next_word_pos, next_word := s.NextWord(idx[1]) if len(next_word) > 3 { for _, r := range next_word { if !unicode.IsUpper(r) && unicode.IsLetter(r) { return false } if unicode.IsSpace(r) { return false } } // mc, donalds := s.PreviousRune(next_word_pos[0]); countme := 0 for unicode.IsSpace(mc) { mc, donalds = s.PreviousRune(donalds) countme++ } if countme > 1 { return false } // } next, _ = s.NextRune(next_word_pos[1]) if next == 46 { // Period, dot or full stop return true } return false } func (s *Searcher) NextWord(idx int) ([]int, []rune) { r, idx := s.NextRune(idx) for !unicode.IsSpace(r) { r, idx = s.NextRune(idx) } for !unicode.IsLetter(r) { r, idx = s.NextRune(idx) } _, start := s.PreviousRune(idx) for !unicode.IsSpace(r) { r, idx = s.NextRune(idx) } r, idx = s.PreviousRune(idx) for !unicode.IsLetter(r) { r, idx = s.PreviousRune(idx) } _, end := s.NextRune(idx) runes := make([]rune, 0) for _, r := range s.CompleteWorks[start:end] { runes = append(runes, r) } return []int{start, end}, runes } func (s *Searcher) PreviousWord(idx int) ([]int, []rune) { // TODO: 30542 handle all caps r, idx := s.PreviousRune(idx) for !unicode.IsLetter(r) { r, idx = s.PreviousRune(idx) } _, end := s.NextRune(idx) for !unicode.IsSpace(r) { r, idx = s.PreviousRune(idx) } _, start := s.NextRune(idx) runes := make([]rune, 0) for _, r := range s.CompleteWorks[start:end] { runes = append(runes, r) } return []int{start, end}, runes } func (s *Searcher) NextBlock(idx int) []int { return []int{1, 2} } func (s *Searcher) PreviousBlock(idx int) []int { return []int{1, 2} } func (s *Searcher) TitleThis(idx int) string { r, pos := s.PreviousRune(idx) for !unicode.IsLetter(r) { r, pos = s.PreviousRune(pos) } _, end := s.NextRune(pos) for r != 10 { r, pos = s.PreviousRune(pos) } _, start := s.NextRune(pos) for _, r := range s.CompleteWorks[start:end] { if unicode.IsLetter(r) { break } if unicode.IsSpace(r) { _, start = s.NextRune(start) } } return s.CompleteWorks[start:end] } func (s *Searcher) CountBytesToNewline(pos int) string { i := 0 nl := 0 for i = range s.CompleteWorks { if s.CompleteWorks[i] == 10 { nl++ } if nl == pos { break } } return strconv.Itoa(i) } func (s *Searcher) CountNewlinesToByte(idx int) string { nl := 1 for i, r := range s.CompleteWorks { if r == 10 { nl++ } if i >= idx { break } } return strconv.Itoa(nl * 30) // 30px line height } func (s *Searcher) doMath() { total := 0 for i := range pages { total = total + (pages[i] * 2800) } fmt.Println("total: ", total) fmt.Println("len: ", len(s.CompleteWorks)) difference := len(s.CompleteWorks) - total fmt.Println("difference: ", difference) } func (s *Searcher) findContext(idx []int) ([]int, string) { // ... 30543, TODO: 135989 (all caps, no period, end) title := "N/A" for _, x := range BYTE_RANGE { if idx[0] > x[0] && idx[1] < x[1] { title = s.TitleThis(x[0]) } } i, r := s.PreviousWord(idx[0]) for !s.Speaker(r, i) { i, r = s.PreviousWord(i[0]) } BLOCK_START := i[0] i, r = s.NextWord(idx[1]) for !s.Speaker(r, i) { // TODO: check here i, r = s.NextWord(i[1]) } rune, pos := s.PreviousRune(i[0]) for !unicode.IsSpace(rune) { rune, pos = s.PreviousRune(pos) } BLOCK_END := pos return []int{BLOCK_START, BLOCK_END}, title } func isUpper(s string) bool { letter := false for _, r := range s { if !unicode.IsUpper(r) && unicode.IsLetter(r) { return false } if unicode.IsLetter(r) { letter = true } } if letter == true { return true } else { return false } } func (s *Searcher) Webify() { bucket := make([]rune, 0) i := 1 start := true head := []rune(`
") for _, r := range s.CompleteWorks { if r == 10 { for m := range tail { bucket = append(bucket, tail[m]) } i++ } if start == true { for z := range head { bucket = append(bucket, head[z]) } iz := strconv.Itoa(i) for _, bb := range iz { bucket = append(bucket, bb) } tmp := []rune(`">`) for q := range tmp { bucket = append(bucket, tmp[q]) } start = false } bucket = append(bucket, r) if r == 10 { start = true } } file, err := os.Create("webify.txt") if err != nil { fmt.Println(err) } defer file.Close() _, err = io.WriteString(file, string(bucket)) if err != nil { fmt.Println(err) } err = file.Sync() if err != nil { fmt.Println(err) } } func (s *Searcher) Search(query string) []string { // TODO: define words better, e.g. disease,—my project may re, _ := regexp.Compile("(?im)" + query) idxs := s.SuffixArray.FindAllIndex(re, -1) results := []string{} for _, idx := range idxs { if idx[0] > 2919 && idx[1] < 5745637 { // works are in this range if isUpper(s.CompleteWorks[idx[0]:idx[1]]) { //break } else { idx, title := s.findContext(idx) // maybe change later - can use title start as position for view anchor := s.CountNewlinesToByte(idx[0]) results = append(results, ``+s.CompleteWorks[idx[0]:idx[1]]+"

"+title+"

") // embed anchor here } } } return results }