moved event store definition to crawler from relay

2025-12-17 07:24:21 +01:00 · 2025-07-22 14:21:38 +02:00
parent ec5a9a3162
commit cce82de057
6 changed files with 159 additions and 17 deletions
--- a/pkg/store/store.go
+++ b/pkg/store/store.go
@@ -0,0 +1,84 @@
+package store
+
+import (
+	"github.com/pippellia-btc/nastro/sqlite"
+)
+
+var (
+	profileFTS = `
+	CREATE VIRTUAL TABLE IF NOT EXISTS profiles_fts USING fts5(
+		id UNINDEXED,
+		pubkey UNINDEXED,
+		name,
+		display_name,
+		about,
+		website,
+		nip05,
+		tokenize = 'trigram',
+	  );
+
+	CREATE TRIGGER IF NOT EXISTS profiles_ai AFTER INSERT ON events
+	  WHEN NEW.kind = 0
+	  BEGIN
+	  INSERT INTO profiles_fts (id, pubkey, name, display_name, about, website, nip05)
+	  VALUES (
+	  	NEW.id,
+	  	NEW.pubkey,
+	  	NEW.content ->> '$.name',
+	  	COALESCE( NEW.content ->> '$.display_name', NEW.content ->> '$.displayName'),
+	  	NEW.content ->> '$.about',
+	  	NEW.content ->> '$.website',
+	  	NEW.content ->> '$.nip05'
+	  );
+	  END;
+
+	CREATE TRIGGER IF NOT EXISTS profiles_ad AFTER DELETE ON events
+	WHEN OLD.kind = 0
+	BEGIN
+	  DELETE FROM profiles_fts WHERE id = OLD.id;
+	END;`
+
+	// indexing a-z and A-Z tags of responses for efficient look-up
+	responseTagsIndex = `
+	CREATE TRIGGER IF NOT EXISTS response_tags_ai AFTER INSERT ON events
+	WHEN (NEW.kind BETWEEN 6312 AND 6315 OR NEW.kind = 7000)
+	BEGIN
+	INSERT INTO event_tags (event_id, key, value)
+		SELECT NEW.id, json_extract(value, '$[0]'), json_extract(value, '$[1]')
+		FROM json_each(NEW.tags)
+		WHERE json_type(value) = 'array'
+			AND json_array_length(value) > 1
+			AND typeof(json_extract(value, '$[0]')) = 'text'
+			AND json_extract(value, '$[0]') GLOB '[a-zA-Z]';
+	END;`
+)
+
+func New(URL string, opts ...sqlite.Option) (*sqlite.Store, error) {
+	store, err := sqlite.New(URL,
+		sqlite.WithAdditionalSchema(profileFTS),
+		sqlite.WithAdditionalSchema(responseTagsIndex),
+		sqlite.WithRetries(2),
+	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	for _, opt := range opts {
+		if err := opt(store); err != nil {
+			return nil, err
+		}
+	}
+	return store, nil
+}
+
+// Profile represent the internal representation of the content of kind:0s, used for full-text-search.
+type Profile struct {
+	ID          string
+	Pubkey      string
+	Name        string
+	DisplayName string
+	About       string
+	Website     string
+	Nip05       string
+}
--- a/pkg/store/store_test.go
+++ b/pkg/store/store_test.go
@@ -0,0 +1,64 @@
+package store
+
+import (
+	"context"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/nbd-wtf/go-nostr"
+)
+
+var (
+	ctx = context.Background()
+	URL = "test.sqlite"
+
+	event = nostr.Event{
+		ID:        "f7a73d54e45714f5e3ca97b789dfc7898e7dd31f77981989d71a54030e627ff6",
+		Kind:      0,
+		PubKey:    "f683e87035f7ad4f44e0b98cfbd9537e16455a92cd38cefc4cb31db7557f5ef2",
+		CreatedAt: 1739547448,
+		Sig:       "51a89ee1e24d83bd8e9209daf6a38245c974b49206ecb66fe156c9d7875c782f653b40cd73582f6bc9de5d1db497b925a13a828d521f8b78982fea359206e4e8",
+		Content:   "{\"name\":\"pippellia\",\"nip05\":\"pip@vertexlab.io\",\"about\":\"simplifying social graph analysis so you can focus on building great experiences https://vertexlab.io/\",\"lud16\":\"whitebat1@primal.net\",\"display_name\":\"Pip the social graph guy\",\"picture\":\"https://m.primal.net/IfSZ.jpg\",\"banner\":\"https://m.primal.net/IfSc.png\",\"website\":\"pippellia.com\",\"displayName\":\"Pip the social graph guy\",\"pubkey\":\"f683e87035f7ad4f44e0b98cfbd9537e16455a92cd38cefc4cb31db7557f5ef2\",\"npub\":\"npub176p7sup477k5738qhxx0hk2n0cty2k5je5uvalzvkvwmw4tltmeqw7vgup\",\"created_at\":1738783677}",
+	}
+
+	profile = Profile{
+		ID:          event.ID,
+		Pubkey:      event.PubKey,
+		Name:        "pippellia",
+		DisplayName: "Pip the social graph guy",
+		About:       "simplifying social graph analysis so you can focus on building great experiences https://vertexlab.io/",
+		Website:     "pippellia.com",
+		Nip05:       "pip@vertexlab.io",
+	}
+)
+
+func TestSaveProfile(t *testing.T) {
+	store, err := New(URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer Remove(URL)
+
+	if err := store.Save(ctx, &event); err != nil {
+		t.Fatal(err)
+	}
+
+	var p Profile
+	row := store.DB.QueryRowContext(ctx, "SELECT * FROM profiles_fts WHERE id = ?", event.ID)
+	err = row.Scan(&p.ID, &p.Pubkey, &p.Name, &p.DisplayName, &p.About, &p.Website, &p.Nip05)
+
+	if err != nil {
+		t.Fatalf("failed to query for event ID %s in profiles_fts: %v", event.ID, err)
+	}
+
+	if !reflect.DeepEqual(p, profile) {
+		t.Fatalf("expected profile %v, got %v", profile, p)
+	}
+}
+
+func Remove(URL string) {
+	os.Remove(URL)
+	os.Remove(URL + "-shm")
+	os.Remove(URL + "-wal")
+}