sour-is/ev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: move apps to go.sour.is/tools

Browse Source
This commit is contained in:
xuu
2023-09-29 10:07:24 -06:00
parent 813c2e898d
commit ee45a0fd49
98 changed files with 35 additions and 12955 deletions
Download Patch File Download Diff File Expand all files Collapse all files

402
driver/disk-store/disk-store.go Normal file
View File

@@ -0,0 +1,402 @@
// package diskstore provides a driver that reads and writes events to disk.
package diskstore
import (
"context"
"errors"
"fmt"
"hash/fnv"
"os"
"path/filepath"
"strings"
"github.com/tidwall/wal"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"go.uber.org/multierr"
"go.sour.is/pkg/cache"
"go.sour.is/pkg/lg"
"go.sour.is/pkg/locker"
"go.sour.is/ev"
"go.sour.is/ev/driver"
"go.sour.is/ev/event"
)
const CachSize = 1000
const AppendOnly = ev.AppendOnly
const AllEvents = ev.AllEvents
type lockedWal = locker.Locked[wal.Log]
type openlogs struct {
logs *cache.Cache[string, *lockedWal]
}
type diskStore struct {
path string
openlogs *locker.Locked[openlogs]
m_disk_open metric.Int64Counter
m_disk_evict metric.Int64Counter
m_disk_read metric.Int64Counter
m_disk_write metric.Int64Counter
}
var _ driver.Driver = (*diskStore)(nil)
func Init(ctx context.Context) error {
ctx, span := lg.Span(ctx)
defer span.End()
d := &diskStore{}
m := lg.Meter(ctx)
var err, errs error
d.m_disk_open, err = m.Int64Counter("disk_open")
errs = multierr.Append(errs, err)
d.m_disk_evict, err = m.Int64Counter("disk_evict")
errs = multierr.Append(errs, err)
d.m_disk_read, err = m.Int64Counter("disk_read")
errs = multierr.Append(errs, err)
d.m_disk_write, err = m.Int64Counter("disk_write")
errs = multierr.Append(errs, err)
ev.Register(ctx, "file", d)
return errs
}
func (d *diskStore) Open(ctx context.Context, dsn string) (driver.Driver, error) {
_, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.String("args.dsn", dsn),
)
scheme, path, ok := strings.Cut(dsn, ":")
if !ok {
return nil, fmt.Errorf("expected scheme")
}
if scheme != "file" {
return nil, fmt.Errorf("expeted scheme=file, got=%s", scheme)
}
if _, err := os.Stat(path); os.IsNotExist(err) {
err = os.MkdirAll(path, 0700)
if err != nil {
span.RecordError(err)
return nil, err
}
}
c, err := cache.NewWithEvict(CachSize, func(ctx context.Context, s string, l *lockedWal) {
ctx, span := lg.Span(ctx)
defer span.End()
l.Use(ctx, func(ctx context.Context, w *wal.Log) error {
ctx, span := lg.Span(ctx)
defer span.End()
d.m_disk_evict.Add(ctx, 1)
err := w.Close()
if err != nil {
span.RecordError(err)
return err
}
return nil
})
})
if err != nil {
span.RecordError(err)
return nil, err
}
logs := &openlogs{logs: c}
return &diskStore{
path: path,
openlogs: locker.New(logs),
m_disk_open: d.m_disk_open,
m_disk_evict: d.m_disk_evict,
m_disk_read: d.m_disk_read,
m_disk_write: d.m_disk_write,
}, nil
}
func (d *diskStore) EventLog(ctx context.Context, streamID string) (driver.EventLog, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.String("args.streamID", streamID),
attribute.String("path", d.path),
)
el := &eventLog{streamID: streamID, diskStore: d}
return el, d.openlogs.Use(ctx, func(ctx context.Context, openlogs *openlogs) error {
ctx, span := lg.Span(ctx)
defer span.End()
if events, ok := openlogs.logs.Get(streamID); ok {
el.events = *events
return nil
}
d.m_disk_open.Add(ctx, 1)
// migrate streams into dir friendly subdirs
hashPart := mkDirName(streamID)
oldPath := filepath.Join(d.path, streamID)
newPath := filepath.Join(d.path, hashPart, streamID)
if _, err := os.Stat(oldPath); !os.IsNotExist(err) {
os.MkdirAll(filepath.Join(d.path, hashPart), 0700)
os.Rename(oldPath, newPath)
}
l, err := wal.Open(newPath, wal.DefaultOptions)
if err != nil {
span.RecordError(err)
return err
}
el.events = locker.New(l)
openlogs.logs.Add(ctx, streamID, el.events)
return nil
})
}
type eventLog struct {
streamID string
events *locker.Locked[wal.Log]
diskStore *diskStore
}
var _ driver.EventLog = (*eventLog)(nil)
func (e *eventLog) Append(ctx context.Context, events event.Events, version uint64) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int("args.events", len(events)),
attribute.Int64("args.version", int64(version)),
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
event.SetStreamID(e.streamID, events...)
var count uint64
err := e.events.Use(ctx, func(ctx context.Context, l *wal.Log) error {
ctx, span := lg.Span(ctx)
defer span.End()
last, err := l.LastIndex()
if err != nil {
span.RecordError(err)
return err
}
if version != AppendOnly && version != last {
err = fmt.Errorf("%w: current version wrong %d != %d", ev.ErrWrongVersion, version, last)
span.RecordError(err)
return err
}
var b []byte
batch := &wal.Batch{}
for i, e := range events {
span.AddEvent(fmt.Sprintf("append event %d of %d", i, len(events)))
b, err = event.MarshalBinary(e)
if err != nil {
span.RecordError(err)
return err
}
pos := last + uint64(i) + 1
event.SetPosition(e, pos)
batch.Write(pos, b)
}
count = uint64(len(events))
e.diskStore.m_disk_write.Add(ctx, int64(len(events)))
return l.WriteBatch(batch)
})
span.RecordError(err)
return count, err
}
func (e *eventLog) ReadN(ctx context.Context, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
lis := make([]int64, len(index))
for i := range index {
lis[i] = int64(index[i])
}
span.SetAttributes(
attribute.Int64Slice("args.index", lis),
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
var events event.Events
err := e.events.Use(ctx, func(ctx context.Context, stream *wal.Log) error {
var err error
events, err = readStreamN(ctx, stream, index...)
return err
})
return events, err
}
func (e *eventLog) Read(ctx context.Context, after, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.after", after),
attribute.Int64("args.count", count),
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
var events event.Events
err := e.events.Use(ctx, func(ctx context.Context, stream *wal.Log) error {
ctx, span := lg.Span(ctx)
defer span.End()
first, err := stream.FirstIndex()
if err != nil {
return err
}
last, err := stream.LastIndex()
if err != nil {
return err
}
streamIDs, err := driver.GenerateStreamIDs(first, last, after, count)
if err != nil {
return err
}
events, err = readStreamN(ctx, stream, streamIDs...)
event.SetStreamID(e.streamID, events...)
return err
})
if err != nil {
span.RecordError(err)
return nil, err
}
e.diskStore.m_disk_read.Add(ctx, int64(len(events)))
return events, nil
}
func (e *eventLog) FirstIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
var idx uint64
var err error
err = e.events.Use(ctx, func(ctx context.Context, events *wal.Log) error {
idx, err = events.FirstIndex()
return err
})
return idx, err
}
func (e *eventLog) LastIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
var idx uint64
var err error
err = e.events.Use(ctx, func(ctx context.Context, events *wal.Log) error {
idx, err = events.LastIndex()
return err
})
return idx, err
}
func (e *eventLog) Truncate(ctx context.Context, index int64) error {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.index", index),
attribute.String("streamID", e.streamID),
attribute.String("path", e.diskStore.path),
)
if index == 0 {
return nil
}
return e.events.Use(ctx, func(ctx context.Context, events *wal.Log) error {
if index < 0 {
return events.TruncateBack(uint64(-index))
}
return events.TruncateFront(uint64(index))
})
}
func readStreamN(ctx context.Context, stream *wal.Log, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
lis := make([]int64, len(index))
for i := range index {
lis[i] = int64(index[i])
}
span.SetAttributes(
attribute.Int64Slice("args.index", lis),
)
var b []byte
var err error
events := make(event.Events, len(index))
for i, idx := range index {
b, err = stream.Read(idx)
if err != nil {
if errors.Is(err, wal.ErrNotFound) || errors.Is(err, wal.ErrOutOfRange) {
err = fmt.Errorf("%w: empty", ev.ErrNotFound)
}
span.RecordError(err)
return nil, err
}
events[i], err = event.UnmarshalBinary(ctx, b, idx)
if err != nil {
span.RecordError(err)
return nil, err
}
span.AddEvent(fmt.Sprintf("read event %d of %d - %d", i, len(events), events[i].EventMeta().ActualPosition))
}
return events, err
}
func mkDirName(name string) string {
h := fnv.New32a()
fmt.Fprint(h, name)
return fmt.Sprintf("%x/%x/%x", h.Sum32()>>24&0xff, h.Sum32()>>16&0xff, h.Sum32()&0xffff)
}

69
driver/driver.go Normal file
View File

@@ -0,0 +1,69 @@
// package driver defines interfaces to be used by driver implementations.
package driver
import (
"context"
"go.sour.is/ev/event"
"go.sour.is/pkg/math"
)
type Driver interface {
Open(ctx context.Context, dsn string) (Driver, error)
EventLog(ctx context.Context, streamID string) (EventLog, error)
}
type EventLog interface {
Read(ctx context.Context, after, count int64) (event.Events, error)
ReadN(ctx context.Context, index ...uint64) (event.Events, error)
Append(ctx context.Context, events event.Events, version uint64) (uint64, error)
FirstIndex(context.Context) (uint64, error)
LastIndex(context.Context) (uint64, error)
}
type EventLogWithTruncate interface {
Truncate(context.Context, int64) error
}
type EventLogWithUpdate interface {
LoadForUpdate(context.Context, event.Aggregate, func(context.Context, event.Aggregate) error) (uint64, error)
}
type Subscription interface {
Recv(context.Context) <-chan bool
Events(context.Context) (event.Events, error)
Close(context.Context) error
}
type EventStream interface {
Subscribe(ctx context.Context, streamID string, start int64) (Subscription, error)
Send(ctx context.Context, streamID string, events event.Events) error
}
func GenerateStreamIDs(first, last uint64, after, count int64) ([]uint64, error) {
// ---
if first == 0 || last == 0 {
return nil, nil
}
start, count := math.PagerBox(first, last, after, count)
if count == 0 {
return nil, nil
}
streamIDs := make([]uint64, math.Abs(count))
for i := range streamIDs {
streamIDs[i] = start
if count > 0 {
start += 1
} else {
start -= 1
}
if start < first || start > last {
streamIDs = streamIDs[:i+1]
break
}
}
return streamIDs, nil
}

239
driver/mem-store/mem-store.go Normal file
View File

@@ -0,0 +1,239 @@
// package memstore provides a driver that reads and writes events to memory.
package memstore
import (
"context"
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.sour.is/pkg/lg"
"go.sour.is/pkg/locker"
"go.sour.is/ev"
"go.sour.is/ev/driver"
"go.sour.is/ev/event"
)
const AppendOnly = ev.AppendOnly
const AllEvents = ev.AllEvents
type state struct {
streams map[string]*locker.Locked[event.Events]
}
type memstore struct {
state *locker.Locked[state]
}
var _ driver.Driver = (*memstore)(nil)
func Init(ctx context.Context) error {
ctx, span := lg.Span(ctx)
defer span.End()
return ev.Register(ctx, "mem", &memstore{})
}
func (memstore) Open(ctx context.Context, name string) (driver.Driver, error) {
_, span := lg.Span(ctx)
defer span.End()
s := &state{streams: make(map[string]*locker.Locked[event.Events])}
return &memstore{locker.New(s)}, nil
}
func (m *memstore) EventLog(ctx context.Context, streamID string) (driver.EventLog, error) {
ctx, span := lg.Span(ctx)
defer span.End()
el := &eventLog{streamID: streamID}
err := m.state.Use(ctx, func(ctx context.Context, state *state) error {
_, span := lg.Span(ctx)
defer span.End()
l, ok := state.streams[streamID]
if !ok {
l = locker.New(&event.Events{})
state.streams[streamID] = l
}
el.events = l
return nil
})
if err != nil {
return nil, err
}
return el, err
}
type eventLog struct {
streamID string
events *locker.Locked[event.Events]
}
var _ driver.EventLog = (*eventLog)(nil)
// Append implements driver.EventStore
func (m *eventLog) Append(ctx context.Context, events event.Events, version uint64) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
event.SetStreamID(m.streamID, events...)
return uint64(len(events)), m.events.Use(ctx, func(ctx context.Context, stream *event.Events) error {
ctx, span := lg.Span(ctx)
defer span.End()
span.AddEvent(fmt.Sprintf(" %s %d", m.streamID, len(*stream)))
last := uint64(len(*stream))
if version != AppendOnly && version != last {
return fmt.Errorf("%w: current version wrong %d != %d", ev.ErrWrongVersion, version, last)
}
for i := range events {
span.AddEvent(fmt.Sprintf("read event %d of %d", i, len(events)))
// --- clone event
e := events[i]
b, err := event.MarshalBinary(e)
if err != nil {
return err
}
e, err = event.UnmarshalBinary(ctx, b, e.EventMeta().Position)
if err != nil {
return err
}
// ---
pos := last + uint64(i) + 1
event.SetPosition(e, pos)
*stream = append(*stream, e)
}
return nil
})
}
// ReadOne implements readone
func (m *eventLog) ReadN(ctx context.Context, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
lis := make([]int64, len(index))
for i := range index {
lis[i] = int64(index[i])
}
span.SetAttributes(
attribute.Int64Slice("args.index", lis),
attribute.String("streamID", m.streamID),
)
var events event.Events
err := m.events.Use(ctx, func(ctx context.Context, stream *event.Events) error {
var err error
events, err = readStreamN(ctx, stream, index...)
return err
})
return events, err
}
// Read implements driver.EventStore
func (m *eventLog) Read(ctx context.Context, after int64, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.after", after),
attribute.Int64("args.count", count),
attribute.String("streamID", m.streamID),
)
var events event.Events
err := m.events.Use(ctx, func(ctx context.Context, stream *event.Events) error {
ctx, span := lg.Span(ctx)
defer span.End()
first := stream.First().EventMeta().Position
last := stream.Last().EventMeta().Position
streamIDs, err := driver.GenerateStreamIDs(first, last, after, count)
if err != nil {
return err
}
events, err = readStreamN(ctx, stream, streamIDs...)
event.SetStreamID(m.streamID, events...)
return err
})
if err != nil {
span.RecordError(err)
return nil, err
}
return events, nil
}
// FirstIndex for the streamID
func (m *eventLog) FirstIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
events, err := m.events.Copy(ctx)
return events.First().EventMeta().Position, err
}
// LastIndex for the streamID
func (m *eventLog) LastIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
events, err := m.events.Copy(ctx)
return events.Last().EventMeta().Position, err
}
func (e *eventLog) Truncate(ctx context.Context, index int64) error {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.Int64("args.index", index),
attribute.String("streamID", e.streamID),
)
if index == 0 {
return nil
}
return e.events.Use(ctx, func(ctx context.Context, events *event.Events) error {
if index < 0 {
*events = (*events)[:index]
return nil
}
*events = (*events)[index:]
return nil
})
}
func readStreamN(ctx context.Context, stream *event.Events, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
var b []byte
var err error
count := len(index)
events := make(event.Events, count)
for i, index := range index {
span.AddEvent(fmt.Sprintf("read event %d of %d", i, count))
e := (*stream)[index-1]
b, err = event.MarshalBinary(e)
if err != nil {
return nil, err
}
events[i], err = event.UnmarshalBinary(ctx, b, e.EventMeta().Position)
if err != nil {
return nil, err
}
}
return events, err
}

153
driver/projecter/projecter.go Normal file
View File

@@ -0,0 +1,153 @@
// package projecter provides a driver middleware to derive new events from other events.
package projecter
import (
"context"
"strings"
"go.sour.is/pkg/lg"
"go.sour.is/ev"
"go.sour.is/ev/driver"
"go.sour.is/ev/event"
)
type projector struct {
up driver.Driver
fns []func(event.Event) []event.Event
}
func New(_ context.Context, fns ...func(event.Event) []event.Event) *projector {
return &projector{fns: fns}
}
func (p *projector) Apply(e *ev.EventStore) {
up := e.Driver
for up != nil {
if op, ok := up.(*projector); ok {
op.AddProjections(p.fns...)
p.up = op.up
return
}
up = ev.Unwrap(up)
}
p.up = e.Driver
e.Driver = p
}
func (s *projector) Unwrap() driver.Driver {
return s.up
}
func (s *projector) Open(ctx context.Context, dsn string) (driver.Driver, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return s.up.Open(ctx, dsn)
}
func (s *projector) EventLog(ctx context.Context, streamID string) (driver.EventLog, error) {
ctx, span := lg.Span(ctx)
defer span.End()
l, err := s.up.EventLog(ctx, streamID)
return &wrapper{l, s}, err
}
func (s *projector) AddProjections(fns ...func(event.Event) []event.Event) {
s.fns = append(s.fns, fns...)
}
type wrapper struct {
up driver.EventLog
projector *projector
}
var _ driver.EventLog = (*wrapper)(nil)
func (r *wrapper) Unwrap() driver.EventLog {
return r.up
}
func (w *wrapper) Read(ctx context.Context, after int64, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.Read(ctx, after, count)
}
func (w *wrapper) ReadN(ctx context.Context, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.ReadN(ctx, index...)
}
func (w *wrapper) Append(ctx context.Context, events event.Events, version uint64) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
i, err := w.up.Append(ctx, events, version)
if err != nil {
return i, err
}
{
ctx, span := lg.Fork(ctx)
go func() {
defer span.End()
var pevents []event.Event
for i := range events {
e := events[i]
for _, fn := range w.projector.fns {
pevents = append(
pevents,
fn(e)...,
)
}
}
for i := range pevents {
e := pevents[i]
l, err := w.projector.up.EventLog(ctx, event.StreamID(e).StreamID())
if err != nil {
span.RecordError(err)
continue
}
_, err = l.Append(ctx, event.NewEvents(e), ev.AppendOnly)
span.RecordError(err)
}
}()
}
return i, err
}
func (w *wrapper) FirstIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.FirstIndex(ctx)
}
func (w *wrapper) LastIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.LastIndex(ctx)
}
func DefaultProjection(e event.Event) []event.Event {
m := e.EventMeta()
streamID := m.StreamID
streamPos := m.Position
eventType := event.TypeOf(e)
pkg, _, _ := strings.Cut(eventType, ".")
e1 := event.NewPtr(streamID, streamPos)
event.SetStreamID("$all", e1)
e2 := event.NewPtr(streamID, streamPos)
event.SetStreamID("$type-"+eventType, e2)
e3 := event.NewPtr(streamID, streamPos)
event.SetStreamID("$pkg-"+pkg, e3)
return []event.Event{e1, e2, e3}
}

137
driver/projecter/projector_test.go Normal file
View File

@@ -0,0 +1,137 @@
package projecter_test
import (
"context"
"testing"
"github.com/matryer/is"
"go.sour.is/ev"
"go.sour.is/ev/driver"
"go.sour.is/ev/driver/projecter"
"go.sour.is/ev/event"
)
type mockDriver struct {
onOpen func(context.Context, string) (driver.Driver, error)
onEventLog func(context.Context, string) (driver.EventLog, error)
}
// Open implements driver.Driver
func (m *mockDriver) Open(ctx context.Context, dsn string) (driver.Driver, error) {
if m.onOpen != nil {
return m.onOpen(ctx, dsn)
}
panic("unimplemented")
}
// EventLog implements driver.Driver
func (m *mockDriver) EventLog(ctx context.Context, streamID string) (driver.EventLog, error) {
if m.onEventLog != nil {
return m.onEventLog(ctx, streamID)
}
panic("unimplemented")
}
var _ driver.Driver = (*mockDriver)(nil)
type mockEventLog struct {
onAppend func(context.Context, event.Events, uint64) (uint64, error)
onFirstIndex func(context.Context) (uint64, error)
onLastIndex func(context.Context) (uint64, error)
onRead func(context.Context, int64, int64) (event.Events, error)
onReadN func(context.Context, ...uint64) (event.Events, error)
}
// Append implements driver.EventLog
func (m *mockEventLog) Append(ctx context.Context, events event.Events, version uint64) (uint64, error) {
if m.onAppend != nil {
return m.onAppend(ctx, events, version)
}
panic("unimplemented")
}
// FirstIndex implements driver.EventLog
func (m *mockEventLog) FirstIndex(ctx context.Context) (uint64, error) {
if m.onFirstIndex != nil {
return m.onFirstIndex(ctx)
}
panic("unimplemented")
}
// LastIndex implements driver.EventLog
func (m *mockEventLog) LastIndex(ctx context.Context) (uint64, error) {
if m.onLastIndex != nil {
return m.onLastIndex(ctx)
}
panic("unimplemented")
}
// Read implements driver.EventLog
func (m *mockEventLog) Read(ctx context.Context, pos int64, count int64) (event.Events, error) {
if m.onRead != nil {
return m.onRead(ctx, pos, count)
}
panic("unimplemented")
}
func (m *mockEventLog) ReadN(ctx context.Context, index ...uint64) (event.Events, error) {
if m.onReadN != nil {
return m.onReadN(ctx, index...)
}
panic("unimplemented")
}
var _ driver.EventLog = (*mockEventLog)(nil)
func TestProjecter(t *testing.T) {
is := is.New(t)
ctx := context.Background()
var events []event.Event
wait := make(chan struct{})
mockEL := &mockEventLog{}
mockEL.onRead = func(ctx context.Context, i1, i2 int64) (event.Events, error) {
return event.NewEvents(), nil
}
mockEL.onAppend = func(ctx context.Context, e event.Events, u uint64) (uint64, error) {
events = append(events, e...)
if wait != nil && len(events) > 3 {
close(wait)
}
return uint64(len(e)), nil
}
mock := &mockDriver{}
mock.onOpen = func(ctx context.Context, s string) (driver.Driver, error) {
return mock, nil
}
mock.onEventLog = func(ctx context.Context, s string) (driver.EventLog, error) {
return mockEL, nil
}
ev.Init(ctx)
ev.Register(ctx, "mock", mock)
es, err := ev.Open(
ctx,
"mock:",
projecter.New(ctx, projecter.DefaultProjection),
)
is.NoErr(err)
_, err = es.Read(ctx, "test", 0, 1)
is.NoErr(err)
_, err = es.Append(ctx, "test", event.NewEvents(event.NilEvent))
is.NoErr(err)
<-wait
is.Equal(len(events), 4)
}

133
driver/resolve-links/resolve-links.go Normal file
View File

@@ -0,0 +1,133 @@
package resolvelinks
import (
"context"
"errors"
"go.sour.is/ev"
"go.sour.is/ev/driver"
"go.sour.is/ev/event"
"go.sour.is/pkg/lg"
)
type resolvelinks struct {
up driver.Driver
}
func New() *resolvelinks {
return &resolvelinks{}
}
func (r *resolvelinks) Apply(es *ev.EventStore) {
r.up = es.Driver
es.Driver = r
}
func (r *resolvelinks) Unwrap() driver.Driver {
return r.up
}
func (r *resolvelinks) Open(ctx context.Context, dsn string) (driver.Driver, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return r.up.Open(ctx, dsn)
}
func (r *resolvelinks) EventLog(ctx context.Context, streamID string) (driver.EventLog, error) {
ctx, span := lg.Span(ctx)
defer span.End()
l, err := r.up.EventLog(ctx, streamID)
return &wrapper{l, r}, err
}
type wrapper struct {
up driver.EventLog
resolver *resolvelinks
}
func (r *wrapper) Unwrap() driver.EventLog {
return r.up
}
func (w *wrapper) Read(ctx context.Context, after int64, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
events, err := w.up.Read(ctx, after, count)
if err != nil {
return nil, err
}
err = w.resolvelinks(ctx, events)
return events, err
}
func (w *wrapper) ReadN(ctx context.Context, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
events, err := w.up.ReadN(ctx, index...)
if err != nil {
return nil, err
}
err = w.resolvelinks(ctx, events)
return events, err
}
func (w *wrapper) Append(ctx context.Context, events event.Events, version uint64) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.Append(ctx, events, version)
}
func (w *wrapper) FirstIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.FirstIndex(ctx)
}
func (w *wrapper) LastIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.LastIndex(ctx)
}
func (w *wrapper) resolvelinks(ctx context.Context, events event.Events) error {
idx := make(map[string][]uint64)
ptrs := make(map[string][]int)
for i := range events {
e := events[i]
if e, ok := e.(*event.EventPtr); ok {
idx[e.StreamID] = append(idx[e.StreamID], e.Pos)
ptrs[e.StreamID] = append(ptrs[e.StreamID], i)
}
}
for streamID, ids := range idx {
d, err := w.resolver.EventLog(ctx, streamID)
if err != nil {
return err
}
ptr := ptrs[streamID]
lis, err := d.ReadN(ctx, ids...)
if err != nil && !errors.Is(err, ev.ErrNotFound) {
return err
}
for i := range lis {
meta := lis[i].EventMeta()
actual := events[ptr[i]].EventMeta()
meta.ActualPosition = actual.Position
meta.ActualStreamID = actual.ActualStreamID
lis[i].SetEventMeta(meta)
events[i] = lis[i]
}
}
return nil
}

328
driver/streamer/streamer.go Normal file
View File

@@ -0,0 +1,328 @@
// package streamer provides a driver to allow awaiting for new events to be added to a stream.
package streamer
import (
"context"
"fmt"
"sync"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.sour.is/pkg/lg"
"go.sour.is/pkg/locker"
"go.sour.is/ev"
"go.sour.is/ev/driver"
"go.sour.is/ev/event"
)
type state struct {
subscribers map[string][]*subscription
}
type streamer struct {
state *locker.Locked[state]
up driver.Driver
}
func New(ctx context.Context) *streamer {
_, span := lg.Span(ctx)
defer span.End()
return &streamer{state: locker.New(&state{subscribers: map[string][]*subscription{}})}
}
var _ ev.Option = (*streamer)(nil)
func (s *streamer) Apply(e *ev.EventStore) {
s.up = e.Driver
e.Driver = s
}
func (s *streamer) Unwrap() driver.Driver {
return s.up
}
var _ driver.Driver = (*streamer)(nil)
func (s *streamer) Open(ctx context.Context, dsn string) (driver.Driver, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return s.up.Open(ctx, dsn)
}
func (s *streamer) EventLog(ctx context.Context, streamID string) (driver.EventLog, error) {
ctx, span := lg.Span(ctx)
defer span.End()
l, err := s.up.EventLog(ctx, streamID)
return &wrapper{streamID, l, s}, err
}
var _ driver.EventStream = (*streamer)(nil)
// Subscribe sets up a subscription that tracks the last HWM. when new events arrive they are notified via Send.
// Call Close when done to clean up allocation.
func (s *streamer) Subscribe(ctx context.Context, streamID string, start int64) (driver.Subscription, error) {
ctx, span := lg.Span(ctx)
defer span.End()
events, err := s.up.EventLog(ctx, streamID)
if err != nil {
return nil, err
}
sub := &subscription{topic: streamID, events: events}
sub.position = locker.New(&position{
idx: start,
size: ev.AllEvents,
})
sub.unsub = s.delete(streamID, sub)
return sub, s.state.Use(ctx, func(ctx context.Context, state *state) error {
state.subscribers[streamID] = append(state.subscribers[streamID], sub)
return nil
})
}
// Send will notify all Recv-ing subscribers that new events are ready.
func (s *streamer) Send(ctx context.Context, streamID string, events event.Events) error {
ctx, span := lg.Span(ctx)
defer span.End()
return s.state.Use(ctx, func(ctx context.Context, state *state) error {
ctx, span := lg.Span(ctx)
defer span.End()
span.AddEvent(fmt.Sprint("subscribers=", len(state.subscribers[streamID])))
for _, sub := range state.subscribers[streamID] {
err := sub.position.Use(ctx, func(ctx context.Context, position *position) error {
ctx, span := lg.Span(ctx)
defer span.End()
span.SetAttributes(
attribute.String("streamID", streamID),
attribute.Int64("actualPosition", int64(events.Last().EventMeta().ActualPosition)),
attribute.String("actualStreamID", events.Last().EventMeta().ActualStreamID),
attribute.Int64("position", int64(events.Last().EventMeta().Position)),
)
position.size = int64(events.Last().EventMeta().ActualPosition - uint64(position.idx))
if position.wait != nil {
close(position.wait)
position.link = trace.LinkFromContext(ctx, attribute.String("src", "event"))
position.wait = nil
}
return nil
})
if err != nil {
return err
}
}
return nil
})
}
func (s *streamer) delete(streamID string, sub *subscription) func(context.Context) error {
return func(ctx context.Context) error {
ctx, span := lg.Span(ctx)
defer span.End()
if err := ctx.Err(); err != nil {
return err
}
return s.state.Use(ctx, func(ctx context.Context, state *state) error {
_, span := lg.Span(ctx)
defer span.End()
lis := state.subscribers[streamID]
for i := range lis {
if lis[i] == sub {
lis[i] = lis[len(lis)-1]
state.subscribers[streamID] = lis[:len(lis)-1]
return nil
}
}
return nil
})
}
}
type wrapper struct {
topic string
up driver.EventLog
streamer *streamer
}
var _ driver.EventLog = (*wrapper)(nil)
func (r *wrapper) Unwrap() driver.EventLog {
return r.up
}
func (w *wrapper) Read(ctx context.Context, after int64, count int64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.Read(ctx, after, count)
}
func (w *wrapper) ReadN(ctx context.Context, index ...uint64) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.ReadN(ctx, index...)
}
func (w *wrapper) Append(ctx context.Context, events event.Events, version uint64) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
i, err := w.up.Append(ctx, events, version)
if err != nil {
return i, err
}
ctx, span = lg.Fork(ctx)
go func() {
defer span.End()
err := w.streamer.Send(ctx, w.topic, events)
span.RecordError(err)
}()
return i, nil
}
func (w *wrapper) FirstIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.FirstIndex(ctx)
}
func (w *wrapper) LastIndex(ctx context.Context) (uint64, error) {
ctx, span := lg.Span(ctx)
defer span.End()
return w.up.LastIndex(ctx)
}
type position struct {
size int64
idx int64
link trace.Link
wait chan struct{}
}
type subscription struct {
topic string
position *locker.Locked[position]
events driver.EventLog
unsub func(context.Context) error
once sync.Once
}
// Recv checks for new events. Returned chan will block until an event is ready or cancelled.
func (s *subscription) Recv(ctx context.Context) <-chan bool {
ctx, span := lg.Span(ctx)
defer span.End()
done := make(chan bool)
go func() {
var wait func(context.Context) bool
defer close(done)
err := s.position.Use(ctx, func(ctx context.Context, position *position) error {
_, span := lg.Span(ctx)
defer span.End()
if position.size == ev.AllEvents {
return nil
}
if position.size == 0 {
position.wait = make(chan struct{})
wait = func(ctx context.Context) bool {
ctx, span := lg.Span(ctx)
defer span.End()
select {
case <-position.wait:
if position.link.SpanContext.IsValid() {
_, span := lg.Span(ctx, trace.WithLinks(position.link))
span.AddEvent("recv event")
span.End()
position.link = trace.Link{}
}
return true
case <-ctx.Done():
return false
}
}
}
position.idx += position.size
position.size = 0
return nil
})
if err != nil {
done <- false
return
}
if wait != nil {
done <- wait(ctx)
return
}
done <- true
}()
return done
}
// Events returns a batch of events since last call. Updates position of HWM to end of stream. Call Recv to be notified when more are ready.
func (s *subscription) Events(ctx context.Context) (event.Events, error) {
ctx, span := lg.Span(ctx)
defer span.End()
var events event.Events
return events, s.position.Use(ctx, func(ctx context.Context, position *position) error {
ctx, span := lg.Span(ctx)
defer span.End()
var err error
events, err = s.events.Read(ctx, position.idx, position.size)
if err != nil {
return err
}
position.size = int64(len(events))
if len(events) > 0 {
position.idx = int64(events.First().EventMeta().ActualPosition - 1)
}
span.SetAttributes(
attribute.Int64("position.idx", position.idx),
attribute.Int64("position.size", position.size),
attribute.Int64("meta.ActualPosition", int64(events.First().EventMeta().ActualPosition)),
attribute.Int64("meta.Position", int64(events.First().EventMeta().Position)),
)
return err
})
}
// Close unsubscribes from a stream
func (s *subscription) Close(ctx context.Context) error {
ctx, span := lg.Span(ctx)
defer span.End()
if s == nil || s.unsub == nil {
return nil
}
var err error
s.once.Do(func() { err = s.unsub(ctx) })
return err
}