sour-is/ev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove duplicate code and fmt

Browse Source
This commit is contained in:
xuu
2023-07-22 08:52:15 -06:00
parent 0f665d3484
commit 70c6996f39
16 changed files with 229 additions and 306 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
pkg/driver/disk-store/disk-store.go
View File

@@ -19,7 +19,6 @@ import (
"go.sour.is/pkg/cache"
"go.sour.is/pkg/lg"
"go.sour.is/pkg/locker"
"go.sour.is/pkg/math"
"go.sour.is/ev"
"go.sour.is/ev/pkg/driver"
@@ -28,6 +27,9 @@ import (
const CachSize = 1000
const AppendOnly = ev.AppendOnly
const AllEvents = ev.AllEvents
type lockedWal = locker.Locked[wal.Log]
type openlogs struct {
logs *cache.Cache[string, *lockedWal]
@@ -42,8 +44,7 @@ type diskStore struct {
m_disk_write metric.Int64Counter
}
const AppendOnly = ev.AppendOnly
const AllEvents = ev.AllEvents
var _ driver.Driver = (*diskStore)(nil)
func Init(ctx context.Context) error {
ctx, span := lg.Span(ctx)
@@ -71,8 +72,6 @@ func Init(ctx context.Context) error {
return errs
}
var _ driver.Driver = (*diskStore)(nil)
func (d *diskStore) Open(ctx context.Context, dsn string) (driver.Driver, error) {
_, span := lg.Span(ctx)
defer span.End()
@@ -237,84 +236,6 @@ func (e *eventLog) Append(ctx context.Context, events event.Events, version uint
return count, err
}
func (e *eventLog) Read(ctx context.Context, after, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.after", after),
attribute.Int64("args.count", count),
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
var events event.Events
err := e.events.Use(ctx, func(ctx context.Context, stream *wal.Log) error {
ctx, span := lg.Span(ctx)
defer span.End()
first, err := stream.FirstIndex()
if err != nil {
span.RecordError(err)
return err
}
last, err := stream.LastIndex()
if err != nil {
span.RecordError(err)
return err
}
// ---
if first == 0 || last == 0 {
return nil
}
start, count := math.PagerBox(first, last, after, count)
if count == 0 {
return nil
}
span.SetAttributes(
attribute.Int64("first", int64(first)),
attribute.Int64("last", int64(last)),
attribute.Int64("start", int64(start)),
attribute.Int64("count", int64(count)),
attribute.Int64("after", int64(after)),
)
events = make([]event.Event, math.Abs(count))
for i := range events {
// ---
events[i], err = readStream(ctx, stream, start)
if err != nil {
span.RecordError(err)
return err
}
// ---
span.AddEvent(fmt.Sprintf("read event %d of %d - %d", i, len(events), events[i].EventMeta().ActualPosition))
if count > 0 {
start += 1
} else {
start -= 1
}
if start < first || start > last {
events = events[:i+1]
break
}
}
return nil
})
if err != nil {
span.RecordError(err)
return nil, err
}
event.SetStreamID(e.streamID, events...)
e.diskStore.m_disk_read.Add(ctx, int64(len(events)))
return events, nil
}
func (e *eventLog) ReadN(ctx context.Context, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
@@ -341,6 +262,46 @@ func (e *eventLog) ReadN(ctx context.Context, index ...uint64) (event.Events, er
return events, err
}
func (e *eventLog) Read(ctx context.Context, after, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.after", after),
attribute.Int64("args.count", count),
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
var events event.Events
err := e.events.Use(ctx, func(ctx context.Context, stream *wal.Log) error {
ctx, span := lg.Span(ctx)
defer span.End()
first, err := stream.FirstIndex()
if err != nil {
return err
}
last, err := stream.LastIndex()
if err != nil {
return err
}
streamIDs, err := driver.GenerateStreamIDs(first, last, after, count)
if err != nil {
return err
}
events, err = readStreamN(ctx, stream, streamIDs...)
event.SetStreamID(e.streamID, events...)
return err
})
if err != nil {
span.RecordError(err)
return nil, err
}
e.diskStore.m_disk_read.Add(ctx, int64(len(events)))
return events, nil
}
func (e *eventLog) FirstIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
@@ -399,32 +360,6 @@ func (e *eventLog) Truncate(ctx context.Context, index int64) error {
return events.TruncateFront(uint64(index))
})
}
func readStream(ctx context.Context, stream *wal.Log, index uint64) (event.Event, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.index", int64(index)),
)
var b []byte
var err error
b, err = stream.Read(index)
if err != nil {
if errors.Is(err, wal.ErrNotFound) || errors.Is(err, wal.ErrOutOfRange) {
err = fmt.Errorf("%w: empty", ev.ErrNotFound)
}
span.RecordError(err)
return nil, err
}
e, err := event.UnmarshalBinary(ctx, b, index)
if err != nil {
span.RecordError(err)
return nil, err
}
return e, err
}
func readStreamN(ctx context.Context, stream *wal.Log, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
@@ -442,6 +377,7 @@ func readStreamN(ctx context.Context, stream *wal.Log, index ...uint64) (event.E
var err error
events := make(event.Events, len(index))
for i, idx := range index {
span.AddEvent(fmt.Sprintf("read event %d of %d - %d", i, len(events), events[i].EventMeta().ActualPosition))
b, err = stream.Read(idx)
if err != nil {
if errors.Is(err, wal.ErrNotFound) || errors.Is(err, wal.ErrOutOfRange) {

29
pkg/driver/driver.go
View File

@@ -5,6 +5,7 @@ import (
"context"
"go.sour.is/ev/pkg/event"
"go.sour.is/pkg/math"
)
type Driver interface {
@@ -38,3 +39,31 @@ type EventStream interface {
Subscribe(ctx context.Context, streamID string, start int64) (Subscription, error)
Send(ctx context.Context, streamID string, events event.Events) error
}
func GenerateStreamIDs(first, last uint64, after, count int64) ([]uint64, error) {
// ---
if first == 0 || last == 0 {
return nil, nil
}
start, count := math.PagerBox(first, last, after, count)
if count == 0 {
return nil, nil
}
streamIDs := make([]uint64, math.Abs(count))
for i := range streamIDs {
streamIDs[i] = start
if count > 0 {
start += 1
} else {
start -= 1
}
if start < first || start > last {
streamIDs = streamIDs[:i+1]
break
}
}
return streamIDs, nil
}

112
pkg/driver/mem-store/mem-store.go
View File

@@ -5,28 +5,26 @@ import (
"context"
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.sour.is/pkg/lg"
"go.sour.is/pkg/locker"
"go.sour.is/pkg/math"
"go.sour.is/ev"
"go.sour.is/ev/pkg/driver"
"go.sour.is/ev/pkg/event"
)
const AppendOnly = ev.AppendOnly
const AllEvents = ev.AllEvents
type state struct {
streams map[string]*locker.Locked[event.Events]
}
type eventLog struct {
streamID string
events *locker.Locked[event.Events]
}
type memstore struct {
state *locker.Locked[state]
}
const AppendOnly = ev.AppendOnly
const AllEvents = ev.AllEvents
var _ driver.Driver = (*memstore)(nil)
func Init(ctx context.Context) error {
ctx, span := lg.Span(ctx)
@@ -35,8 +33,6 @@ func Init(ctx context.Context) error {
return ev.Register(ctx, "mem", &memstore{})
}
var _ driver.Driver = (*memstore)(nil)
func (memstore) Open(ctx context.Context, name string) (driver.Driver, error) {
_, span := lg.Span(ctx)
defer span.End()
@@ -68,6 +64,11 @@ func (m *memstore) EventLog(ctx context.Context, streamID string) (driver.EventL
return el, err
}
type eventLog struct {
streamID string
events *locker.Locked[event.Events]
}
var _ driver.EventLog = (*eventLog)(nil)
// Append implements driver.EventStore
@@ -117,6 +118,16 @@ func (m *eventLog) ReadN(ctx context.Context, index ...uint64) (event.Events, er
ctx, span := lg.Span(ctx)
defer span.End()
lis := make([]int64, len(index))
for i := range index {
lis[i] = int64(index[i])
}
span.SetAttributes(
attribute.Int64Slice("args.index", lis),
attribute.String("streamID", m.streamID),
)
var events event.Events
err := m.events.Use(ctx, func(ctx context.Context, stream *event.Events) error {
var err error
@@ -133,54 +144,30 @@ func (m *eventLog) ReadN(ctx context.Context, index ...uint64) (event.Events, er
func (m *eventLog) Read(ctx context.Context, after int64, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.after", after),
attribute.Int64("args.count", count),
attribute.String("streamID", m.streamID),
)
var events event.Events
err := m.events.Use(ctx, func(ctx context.Context, stream *event.Events) error {
ctx, span := lg.Span(ctx)
defer span.End()
span.AddEvent(fmt.Sprintf("%s %d", m.streamID, len(*stream)))
first := stream.First().EventMeta().Position
last := stream.Last().EventMeta().Position
// ---
if first == 0 || last == 0 {
return nil
}
start, count := math.PagerBox(first, last, after, count)
if count == 0 {
return nil
}
span.AddEvent(fmt.Sprint("box", first, last, after, count))
events = make([]event.Event, math.Abs(count))
for i := range events {
span.AddEvent(fmt.Sprintf("read event %d of %d", i, math.Abs(count)))
// --- clone event
var err error
events[i], err = readStream(ctx, stream, start)
if err != nil {
return err
}
// ---
if count > 0 {
start += 1
} else {
start -= 1
}
if start < first || start > last {
events = events[:i+1]
break
}
streamIDs, err := driver.GenerateStreamIDs(first, last, after, count)
if err != nil {
return err
}
events, err = readStreamN(ctx, stream, streamIDs...)
event.SetStreamID(m.streamID, events...)
return nil
return err
})
if err != nil {
span.RecordError(err)
return nil, err
}
@@ -205,26 +192,26 @@ func (m *eventLog) LastIndex(ctx context.Context) (uint64, error) {
return events.Last().EventMeta().Position, err
}
func (m *eventLog) LoadForUpdate(ctx context.Context, a event.Aggregate, fn func(context.Context, event.Aggregate) error) (uint64, error) {
panic("not implemented")
}
func readStream(ctx context.Context, stream *event.Events, index uint64) (event.Event, error) {
func (e *eventLog) Truncate(ctx context.Context, index int64) error {
ctx, span := lg.Span(ctx)
defer span.End()
var b []byte
var err error
e := (*stream)[index-1]
b, err = event.MarshalBinary(e)
if err != nil {
return nil, err
span.SetAttributes(
attribute.Int64("args.index", index),
attribute.String("streamID", e.streamID),
)
if index == 0 {
return nil
}
e, err = event.UnmarshalBinary(ctx, b, e.EventMeta().ActualPosition)
if err != nil {
return nil, err
}
return e, err
return e.events.Use(ctx, func(ctx context.Context, events *event.Events) error {
if index < 0 {
*events = (*events)[:index]
return nil
}
*events = (*events)[index:]
return nil
})
}
func readStreamN(ctx context.Context, stream *event.Events, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
@@ -233,8 +220,11 @@ func readStreamN(ctx context.Context, stream *event.Events, index ...uint64) (ev
var b []byte
var err error
events := make(event.Events, len(index))
count := len(index)
events := make(event.Events, count)
for i, index := range index {
span.AddEvent(fmt.Sprintf("read event %d of %d", i, count))
e := (*stream)[index-1]
b, err = event.MarshalBinary(e)
if err != nil {

1
pkg/driver/projecter/projecter.go
View File

@@ -21,7 +21,6 @@ func New(_ context.Context, fns ...func(event.Event) []event.Event) *projector {
return &projector{fns: fns}
}
func (p *projector) Apply(e *ev.EventStore) {
up := e.Driver
for up != nil {
if op, ok := up.(*projector); ok {

1
pkg/driver/projecter/projector_test.go
View File

@@ -134,5 +134,4 @@ func TestProjecter(t *testing.T) {
<-wait
is.Equal(len(events), 4)
}

117
pkg/driver/resolve-links/resolve-links.go
View File

@@ -43,8 +43,8 @@ func (r *resolvelinks) EventLog(ctx context.Context, streamID string) (driver.Ev
}
type wrapper struct {
up driver.EventLog
resolvelinks *resolvelinks
up driver.EventLog
resolver *resolvelinks
}
func (r *wrapper) Unwrap() driver.EventLog {
@@ -60,60 +60,7 @@ func (w *wrapper) Read(ctx context.Context, after int64, count int64) (event.Eve
return nil, err
}
idx := make(map[string][]uint64)
ptrs := make(map[string][]int)
for i := range events {
e := events[i]
if e, ok := e.(*event.EventPtr); ok {
idx[e.StreamID] = append(idx[e.StreamID], e.Pos)
ptrs[e.StreamID] = append(ptrs[e.StreamID], i)
}
}
for streamID, ids := range idx {
d, err := w.resolvelinks.EventLog(ctx, streamID)
if err != nil {
return nil, err
}
ptr := ptrs[streamID]
lis, err := d.ReadN(ctx, ids...)
if err != nil && !errors.Is(err, ev.ErrNotFound) {
return nil, err
}
for i := range lis {
meta := lis[i].EventMeta()
actual := events[ptr[i]].EventMeta()
meta.ActualPosition = actual.Position
meta.ActualStreamID = actual.ActualStreamID
lis[i].SetEventMeta(meta)
events[i] = lis[i]
}
}
// for i, e := range events {
// switch e := e.(type) {
// case *event.EventPtr:
// d, err := w.resolvelinks.EventLog(ctx, e.StreamID)
// if err != nil {
// return nil, err
// }
// lis, err := d.ReadN(ctx, e.Pos)
// if err != nil && !errors.Is(err, es.ErrNotFound) {
// return nil, err
// }
// if ne := lis.First(); ne != event.NilEvent {
// meta := ne.EventMeta()
// actual := e.EventMeta()
// meta.ActualPosition = actual.Position
// meta.ActualStreamID = actual.ActualStreamID
// ne.SetEventMeta(meta)
// events[i] = ne
// }
// }
// }
err = w.resolvelinks(ctx, events)
return events, err
}
@@ -126,29 +73,7 @@ func (w *wrapper) ReadN(ctx context.Context, index ...uint64) (event.Events, err
return nil, err
}
for i, e := range events {
switch e := e.(type) {
case *event.EventPtr:
d, err := w.resolvelinks.EventLog(ctx, e.StreamID)
if err != nil {
return nil, err
}
lis, err := d.ReadN(ctx, e.Pos)
if err != nil {
return nil, err
}
ne := lis.First()
meta := ne.EventMeta()
actual := e.EventMeta()
meta.ActualPosition = actual.Position
meta.ActualStreamID = actual.ActualStreamID
ne.SetEventMeta(meta)
events[i] = ne
}
}
err = w.resolvelinks(ctx, events)
return events, err
}
@@ -172,3 +97,37 @@ func (w *wrapper) LastIndex(ctx context.Context) (uint64, error) {
return w.up.LastIndex(ctx)
}
func (w *wrapper) resolvelinks(ctx context.Context, events event.Events) error {
idx := make(map[string][]uint64)
ptrs := make(map[string][]int)
for i := range events {
e := events[i]
if e, ok := e.(*event.EventPtr); ok {
idx[e.StreamID] = append(idx[e.StreamID], e.Pos)
ptrs[e.StreamID] = append(ptrs[e.StreamID], i)
}
}
for streamID, ids := range idx {
d, err := w.resolver.EventLog(ctx, streamID)
if err != nil {
return err
}
ptr := ptrs[streamID]
lis, err := d.ReadN(ctx, ids...)
if err != nil && !errors.Is(err, ev.ErrNotFound) {
return err
}
for i := range lis {
meta := lis[i].EventMeta()
actual := events[ptr[i]].EventMeta()
meta.ActualPosition = actual.Position
meta.ActualStreamID = actual.ActualStreamID
lis[i].SetEventMeta(meta)
events[i] = lis[i]
}
}
return nil
}

9
pkg/driver/streamer/streamer.go
View File

@@ -60,6 +60,8 @@ func (s *streamer) EventLog(ctx context.Context, streamID string) (driver.EventL
var _ driver.EventStream = (*streamer)(nil)
// Subscribe sets up a subscription that tracks the last HWM. when new events arrive they are notified via Send.
// Call Close when done to clean up allocation.
func (s *streamer) Subscribe(ctx context.Context, streamID string, start int64) (driver.Subscription, error) {
ctx, span := lg.Span(ctx)
defer span.End()
@@ -81,6 +83,8 @@ func (s *streamer) Subscribe(ctx context.Context, streamID string, start int64)
return nil
})
}
// Send will notify all Recv-ing subscribers that new events are ready.
func (s *streamer) Send(ctx context.Context, streamID string, events event.Events) error {
ctx, span := lg.Span(ctx)
defer span.End()
@@ -218,6 +222,7 @@ type subscription struct {
once sync.Once
}
// Recv checks for new events. Returned chan will block until an event is ready or cancelled.
func (s *subscription) Recv(ctx context.Context) <-chan bool {
ctx, span := lg.Span(ctx)
defer span.End()
@@ -275,6 +280,8 @@ func (s *subscription) Recv(ctx context.Context) <-chan bool {
return done
}
// Events returns a batch of events since last call. Updates position of HWM to end of stream. Call Recv to be notified when more are ready.
func (s *subscription) Events(ctx context.Context) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
@@ -303,6 +310,8 @@ func (s *subscription) Events(ctx context.Context) (event.Events, error) {
return err
})
}
// Close unsubscribes from a stream
func (s *subscription) Close(ctx context.Context) error {
ctx, span := lg.Span(ctx)
defer span.End()