chore: fix url parse and timing
This commit is contained in:
		
							parent
							
								
									7c0df508f8
								
							
						
					
					
						commit
						d8b87a0072
					
				
							
								
								
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							@ -1,2 +1,3 @@
 | 
				
			|||||||
twt.db*
 | 
					twt.db*
 | 
				
			||||||
feed
 | 
					feed
 | 
				
			||||||
 | 
					__debug*
 | 
				
			||||||
 | 
				
			|||||||
							
								
								
									
										29
									
								
								service.go
									
									
									
									
									
								
							
							
						
						
									
										29
									
								
								service.go
									
									
									
									
									
								
							@ -5,6 +5,7 @@ import (
 | 
				
			|||||||
	"fmt"
 | 
						"fmt"
 | 
				
			||||||
	"io"
 | 
						"io"
 | 
				
			||||||
	"net/http"
 | 
						"net/http"
 | 
				
			||||||
 | 
						"net/url"
 | 
				
			||||||
	"os"
 | 
						"os"
 | 
				
			||||||
	"strings"
 | 
						"strings"
 | 
				
			||||||
	"time"
 | 
						"time"
 | 
				
			||||||
@ -100,7 +101,17 @@ func loadFeed(db *sql.DB, feed io.Reader) error {
 | 
				
			|||||||
	followers := f.Info().GetAll("follow")
 | 
						followers := f.Info().GetAll("follow")
 | 
				
			||||||
	followMap := make(map[string]string, len(followers))
 | 
						followMap := make(map[string]string, len(followers))
 | 
				
			||||||
	for _, f := range f.Info().GetAll("follow") {
 | 
						for _, f := range f.Info().GetAll("follow") {
 | 
				
			||||||
		nick, uri, _ := strings.Cut(f.Value(), " ")
 | 
							nick, uri, ok := strings.Cut(f.Value(), "http")
 | 
				
			||||||
 | 
							if !ok{
 | 
				
			||||||
 | 
								continue
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
							nick = strings.TrimSpace(nick)
 | 
				
			||||||
 | 
							uri = "http" + strings.TrimSpace(uri)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
							if _, err := url.Parse(uri); err != nil {
 | 
				
			||||||
 | 
								continue
 | 
				
			||||||
 | 
							}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
		followMap[nick] = uri
 | 
							followMap[nick] = uri
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -172,7 +183,7 @@ type feed struct {
 | 
				
			|||||||
}
 | 
					}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
func refreshLoop(c console) {
 | 
					func refreshLoop(c console) {
 | 
				
			||||||
	maxInt := int(^uint(0) >> 1)
 | 
						maxInt := 3153600000
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	less := func(a, b *feed) bool {
 | 
						less := func(a, b *feed) bool {
 | 
				
			||||||
		return a.LastScanOn.Time.Before(b.LastScanOn.Time)
 | 
							return a.LastScanOn.Time.Before(b.LastScanOn.Time)
 | 
				
			||||||
@ -202,20 +213,25 @@ func refreshLoop(c console) {
 | 
				
			|||||||
		if !f.LastScanOn.Valid {
 | 
							if !f.LastScanOn.Valid {
 | 
				
			||||||
			f.LastScanOn.Time = time.Now()
 | 
								f.LastScanOn.Time = time.Now()
 | 
				
			||||||
			f.LastScanOn.Valid = true
 | 
								f.LastScanOn.Valid = true
 | 
				
			||||||
 | 
							} else {
 | 
				
			||||||
 | 
								f.LastScanOn.Time = f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
 | 
				
			||||||
		}
 | 
							}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
		f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
 | 
					 | 
				
			||||||
		queue.Insert(&f)
 | 
							queue.Insert(&f)
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
	c.Log("start refresh loop")
 | 
						c.Log("start refresh loop")
 | 
				
			||||||
	for !queue.IsEmpty() {
 | 
						for !queue.IsEmpty() {
 | 
				
			||||||
		f := queue.ExtractMin()
 | 
							f := queue.ExtractMin()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
							c.Log("next", f.URI, "last scan on", f.LastScanOn.Time)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
		select {
 | 
							select {
 | 
				
			||||||
		case <-c.Done():
 | 
							case <-c.Done():
 | 
				
			||||||
			return
 | 
								return
 | 
				
			||||||
		case <-time.After(f.LastScanOn.Time.Sub(time.Now())):
 | 
							case <-time.After(time.Until(f.LastScanOn.Time)):
 | 
				
			||||||
			c.Log("refresh", f.URI)
 | 
								c.Log("refresh", f.URI)
 | 
				
			||||||
		}
 | 
							}
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@ -256,7 +272,8 @@ func refreshLoop(c console) {
 | 
				
			|||||||
 | 
					
 | 
				
			||||||
		db.ExecContext(c.Context, updateFeed, f.LastScanOn, f.RefreshRate, f.ID)
 | 
							db.ExecContext(c.Context, updateFeed, f.LastScanOn, f.RefreshRate, f.ID)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
		f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
 | 
							f.LastScanOn.Time = f.LastScanOn.Time.Add(time.Duration(f.RefreshRate) * time.Second)
 | 
				
			||||||
		queue.Insert(f)
 | 
							c.Log("next scan", f.URI, "on", f.LastScanOn.Time)
 | 
				
			||||||
 | 
							// queue.Insert(f)
 | 
				
			||||||
	}
 | 
						}
 | 
				
			||||||
}
 | 
					}
 | 
				
			||||||
 | 
				
			|||||||
		Loading…
	
	
			
			x
			
			
		
	
		Reference in New Issue
	
	Block a user