trying to remove unused glyphs from mainBuffer since they got merged in emojiBuffer.

go.mod

@@ -5,7 +5,6 @@ go 1.21.4
 require (
 	github.com/PuerkitoBio/goquery v1.5.0
 	github.com/dgraph-io/badger/v4 v4.2.0
-	github.com/fiatjaf/emoji v0.0.1
 	github.com/fiatjaf/eventstore v0.3.3
 	github.com/fiatjaf/khatru v0.2.1
 	github.com/fogleman/gg v1.3.0
@@ -13,8 +12,8 @@ require (
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0
 	github.com/gomarkdown/markdown v0.0.0-20231115200524-a660076da3fd
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/kr/pretty v0.3.1
 	github.com/microcosm-cc/bluemonday v1.0.24
+	github.com/nbd-wtf/emoji v0.0.3
 	github.com/nbd-wtf/go-nostr v0.27.3
 	github.com/nbd-wtf/nostr-sdk v0.0.4
 	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
@@ -57,14 +56,12 @@ require (
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.17.2 // indirect
-	github.com/kr/text v0.2.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/puzpuzpuz/xsync/v2 v2.5.1 // indirect
-	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect
 	github.com/sebest/xff v0.0.0-20210106013422-671bd2870b3a // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
@@ -80,3 +77,5 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/nbd-wtf/emoji => /home/fiatjaf/comp/emoji

go.sum

@@ -40,7 +40,6 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -67,8 +66,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fasthttp/websocket v1.5.3 h1:TPpQuLwJYfd4LJPXvHDYPMFWbLjsT91n3GpWtCQtdek=
 github.com/fasthttp/websocket v1.5.3/go.mod h1:46gg/UBmTU1kUaTcwQXpUxtRwG2PvIZYeA8oL6vF3Fs=
-github.com/fiatjaf/emoji v0.0.1 h1:bVlE1kmUsbaWuSta/ln6fCUs5qX+NkQDHAcYczJGGRQ=
-github.com/fiatjaf/emoji v0.0.1/go.mod h1:GQm0lgpID9NRfmpHm/D8kNKzCKkIoOostPBqPIhOPHg=
 github.com/fiatjaf/eventstore v0.3.3 h1:yZgHecBwHCVU+FuLgwz3gJJu27Xq4P8CNld37mQi9qM=
 github.com/fiatjaf/eventstore v0.3.3/go.mod h1:iqGHNXOMz+ztLHXxG13WBXgz/2bG0q/p1L3olof4dYo=
 github.com/fiatjaf/khatru v0.2.1 h1:NlgjBYH7iJpjFyOJVNEX/E2I1v4d5+KINhA+VxgDr4o=
@@ -155,6 +152,8 @@ github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw=
 github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8=
+github.com/nbd-wtf/emoji v0.0.2 h1:5Bgkh7JOvnW5yAo+QPyFrtXLtPKr4vBgs9RJWlCX2Ho=
+github.com/nbd-wtf/emoji v0.0.2/go.mod h1:tS6D9iI34qwBmWc5g8X7tVDkWXulqbTJRsvsM6QsS88=
 github.com/nbd-wtf/go-nostr v0.27.3 h1:u9fdP5h+Ap3LcDFD2j6F2buU/xOM9ddMY0LCDcC6ZyY=
 github.com/nbd-wtf/go-nostr v0.27.3/go.mod h1:e5WOFsKEpslDOxIgK00NqemH7KuAvKIW6sBXeJYCfUs=
 github.com/nbd-wtf/nostr-sdk v0.0.4 h1:vMCiYpFElKMHPXpZjFVEq4utoTLdTYbkqXVYH1/4uzs=
@@ -174,7 +173,6 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pemistahl/lingua-go v1.4.0 h1:ifYhthrlW7iO4icdubwlduYnmwU37V1sbNrwhKBR4rM=
 github.com/pemistahl/lingua-go v1.4.0/go.mod h1:ECuM1Hp/3hvyh7k8aWSqNCPlTxLemFZsRjocUf3KgME=
-github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -182,7 +180,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/puzpuzpuz/xsync/v2 v2.5.1 h1:mVGYAvzDSu52+zaGyNjC+24Xw2bQi3kTr4QJ6N9pIIU=
 github.com/puzpuzpuz/xsync/v2 v2.5.1/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/cors v1.10.0 h1:62NOS1h+r8p1mW6FM0FSB0exioXLhd/sh15KpjWBZ+8=

image_utils.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"image"
 	"image/color"
 	"image/draw"
@@ -13,7 +14,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/fiatjaf/emoji"
 	"github.com/fogleman/gg"
 	"github.com/go-text/typesetting/di"
 	"github.com/go-text/typesetting/font"
@@ -21,6 +21,7 @@ import (
 	"github.com/go-text/typesetting/language"
 	"github.com/go-text/typesetting/opentype/api"
 	"github.com/go-text/typesetting/shaping"
+	"github.com/nbd-wtf/emoji"
 	"github.com/pemistahl/lingua-go"
 	"github.com/srwiley/rasterx"
 	"golang.org/x/image/math/fixed"
@@ -88,11 +89,11 @@ var (
 		di.DirectionRTL,
 	}
 
-	shaperLock   sync.Mutex
+	shaperLock  sync.Mutex
-	shaperBuffer = harfbuzz.NewBuffer()
+	mainBuffer  = harfbuzz.NewBuffer()
-	emojiBuffer  = harfbuzz.NewBuffer()
+	emojiBuffer = harfbuzz.NewBuffer()
-	fontCache    = make(map[font.Face]*harfbuzz.Font)
+	fontCache   = make(map[font.Face]*harfbuzz.Font)
-	emojiFont    *harfbuzz.Font
+	emojiFont   *harfbuzz.Font
 )
 
 type ScriptRange struct {
@@ -352,7 +353,7 @@ func shortenURLs(text string) string {
 
 // beware: this is all very hacky and I don't know what I am doing!
 // this function is copied from go-text/typesetting/shaping's HarfbuzzShaper and adapted to not require a "class",
-// to rely on our dirty globals like fontCache, shaperLock and shaperBuffer; it also uses a custom function to
+// to rely on our dirty globals like fontCache, shaperLock and mainBuffer; it also uses a custom function to
 // determine language, script, direction and font face internally instead of taking a shaping.Input argument --
 // but also, the most important change was to make it "shape" the same text, twice, with the default font and with
 // the emoji font, then build an output of glyphs containing normal glyphs for when the referenced rune is not an
@@ -364,10 +365,10 @@ func shapeText(rawText []rune, fontSize int) (shaping.Output, []bool) {
 	defer shaperLock.Unlock()
 
 	// load or get main font from cache
-	hfont, ok := fontCache[face]
+	mainFont, ok := fontCache[face]
 	if !ok {
-		hfont = harfbuzz.NewFont(face)
+		mainFont = harfbuzz.NewFont(face)
-		fontCache[face] = hfont
+		fontCache[face] = mainFont
 	}
 
 	// define this only once
@@ -387,7 +388,7 @@ func shapeText(rawText []rune, fontSize int) (shaping.Output, []bool) {
 		font *harfbuzz.Font
 		buf  *harfbuzz.Buffer
 	}{
-		{hfont, shaperBuffer},
+		{mainFont, mainBuffer},
 		{emojiFont, emojiBuffer},
 	} {
 		params.buf.Clear() // clear before using
@@ -413,40 +414,61 @@ func shapeText(rawText []rune, fontSize int) (shaping.Output, []bool) {
 	}
 
 	// this will be used to determine whether a given glyph is an emoji or not when rendering
-	emojiMask := make([]bool, len(shaperBuffer.Info))
+	emojiMask := make([]bool, 0, len(mainBuffer.Info))
 
 	// convert the shaped text into an output
-	glyphs := make([]shaping.Glyph, len(shaperBuffer.Info))
+	glyphs := make([]shaping.Glyph, 0, len(mainBuffer.Info))
-	for i := 0; i < len(glyphs); i++ {
+	g := -1
+	fmt.Printf("[::] %s %X\n", string(rawText), rawText)
+	for i := 0; i < len(mainBuffer.Info); i++ {
+		g++
 		var buf *harfbuzz.Buffer
 		var font *harfbuzz.Font
-		if i < len(rawText) && emoji.IsEmoji(rawText[i]) {
+		if chars, visChars := emoji.GetNextEmojiCharacters(rawText[i:], len(rawText)-i); chars > 0 {
 			buf = emojiBuffer
 			font = emojiFont
-			emojiMask[i] = true
+			emojiMask = append(emojiMask, true)
-		} else {
-			buf = shaperBuffer
-			font = hfont
-		}
-		glyph := buf.Info[i]
 
-		glyphs[i] = shaping.Glyph{
+			// remove the invalid glyphs from mainBuffer
+			fmt.Println(chars, visChars)
+			if chars > 1 {
+				cutN := chars - 1
+
+				for _, g := range mainBuffer.Info[g+1 : len(mainBuffer.Info[g+1+cutN:])] {
+					fmt.Printf("  excluding %s %X \n", string(g.Codepoint), g.Codepoint)
+				}
+
+				copy(mainBuffer.Info[g+1:], mainBuffer.Info[g+1+cutN:])
+				mainBuffer.Info = mainBuffer.Info[0 : len(mainBuffer.Info)-cutN]
+				copy(mainBuffer.Pos[g+1:], mainBuffer.Pos[g+1+cutN:])
+				mainBuffer.Pos = mainBuffer.Pos[0 : len(mainBuffer.Pos)-cutN]
+				i += chars - 1
+			}
+		} else {
+			emojiMask = append(emojiMask, false)
+			buf = mainBuffer
+			font = mainFont
+		}
+
+		glyph := buf.Info[g]
+		glyphs = append(glyphs, shaping.Glyph{
 			ClusterIndex: glyph.Cluster,
 			GlyphID:      glyph.Glyph,
 			Mask:         glyph.Mask,
-		}
+		})
 		extents, ok := font.GlyphExtents(glyph.Glyph)
 		if !ok {
 			continue
 		}
-		glyphs[i].Width = fixed.I(int(extents.Width)) >> scaleShift
+		idx := len(glyphs) - 1
-		glyphs[i].Height = fixed.I(int(extents.Height)) >> scaleShift
+		glyphs[idx].Width = fixed.I(int(extents.Width)) >> scaleShift
-		glyphs[i].XBearing = fixed.I(int(extents.XBearing)) >> scaleShift
+		glyphs[idx].Height = fixed.I(int(extents.Height)) >> scaleShift
-		glyphs[i].YBearing = fixed.I(int(extents.YBearing)) >> scaleShift
+		glyphs[idx].XBearing = fixed.I(int(extents.XBearing)) >> scaleShift
-		glyphs[i].XAdvance = fixed.I(int(buf.Pos[i].XAdvance)) >> scaleShift
+		glyphs[idx].YBearing = fixed.I(int(extents.YBearing)) >> scaleShift
-		glyphs[i].YAdvance = fixed.I(int(buf.Pos[i].YAdvance)) >> scaleShift
+		glyphs[idx].XAdvance = fixed.I(int(buf.Pos[g].XAdvance)) >> scaleShift
-		glyphs[i].XOffset = fixed.I(int(buf.Pos[i].XOffset)) >> scaleShift
+		glyphs[idx].YAdvance = fixed.I(int(buf.Pos[g].YAdvance)) >> scaleShift
-		glyphs[i].YOffset = fixed.I(int(buf.Pos[i].YOffset)) >> scaleShift
+		glyphs[idx].XOffset = fixed.I(int(buf.Pos[g].XOffset)) >> scaleShift
+		glyphs[idx].YOffset = fixed.I(int(buf.Pos[g].YOffset)) >> scaleShift
 	}
 
 	countClusters(glyphs, input.RunEnd, input.Direction.Progression())
@@ -459,7 +481,7 @@ func shapeText(rawText []rune, fontSize int) (shaping.Output, []bool) {
 	out.Runes.Offset = input.RunStart
 	out.Runes.Count = input.RunEnd - input.RunStart
 
-	fontExtents := hfont.ExtentsForDirection(out.Direction.Harfbuzz())
+	fontExtents := mainFont.ExtentsForDirection(out.Direction.Harfbuzz())
 	out.LineBounds = shaping.Bounds{
 		Ascent:  fixed.I(int(fontExtents.Ascender)) >> scaleShift,
 		Descent: fixed.I(int(fontExtents.Descender)) >> scaleShift,
@@ -470,56 +492,16 @@ func shapeText(rawText []rune, fontSize int) (shaping.Output, []bool) {
 	return out, emojiMask
 }
 
-// this function is copied from go-text/typesetting/shaping because shapeText needs it
-func countClusters(glyphs []shaping.Glyph, textLen int, dir di.Progression) {
-	currentCluster := -1
-	runesInCluster := 0
-	glyphsInCluster := 0
-	previousCluster := textLen
-	for i := range glyphs {
-		g := glyphs[i].ClusterIndex
-		if g != currentCluster {
-			// If we're processing a new cluster, count the runes and glyphs
-			// that compose it.
-			runesInCluster = 0
-			glyphsInCluster = 1
-			currentCluster = g
-			nextCluster := -1
-		glyphCountLoop:
-			for k := i + 1; k < len(glyphs); k++ {
-				if glyphs[k].ClusterIndex == g {
-					glyphsInCluster++
-				} else {
-					nextCluster = glyphs[k].ClusterIndex
-					break glyphCountLoop
-				}
-			}
-			if nextCluster == -1 {
-				nextCluster = textLen
-			}
-			switch dir {
-			case di.FromTopLeft:
-				runesInCluster = nextCluster - currentCluster
-			case di.TowardTopLeft:
-				runesInCluster = previousCluster - currentCluster
-			}
-			previousCluster = g
-		}
-		glyphs[i].GlyphCount = glyphsInCluster
-		glyphs[i].RuneCount = runesInCluster
-	}
-}
-
 // this function is copied from go-text/render, but adapted to not require a "class" to be instantiated and also,
 // more importantly, to take an emojiMask parameter, with the same length as out.Glyphs, to determine when a
 // glyph should be rendered with the emoji font instead of with the default font
 func drawShapedRunAt(
+	img draw.Image,
 	fontSize int,
 	clr color.Color,
 	out shaping.Output,
 	emojiMask []bool,
 	maskBaseIndex int,
-	img draw.Image,
 	startX,
 	startY int,
 ) (charsWritten int, endingX int) {
@@ -580,6 +562,46 @@ func drawOutline(g shaping.Glyph, bitmap api.GlyphOutline, f *rasterx.Filler, sc
 	f.Stop(true)
 }
 
+// this function is copied from go-text/typesetting/shaping because shapeText needs it
+func countClusters(glyphs []shaping.Glyph, textLen int, dir di.Progression) {
+	currentCluster := -1
+	runesInCluster := 0
+	glyphsInCluster := 0
+	previousCluster := textLen
+	for i := range glyphs {
+		g := glyphs[i].ClusterIndex
+		if g != currentCluster {
+			// If we're processing a new cluster, count the runes and glyphs
+			// that compose it.
+			runesInCluster = 0
+			glyphsInCluster = 1
+			currentCluster = g
+			nextCluster := -1
+		glyphCountLoop:
+			for k := i + 1; k < len(glyphs); k++ {
+				if glyphs[k].ClusterIndex == g {
+					glyphsInCluster++
+				} else {
+					nextCluster = glyphs[k].ClusterIndex
+					break glyphCountLoop
+				}
+			}
+			if nextCluster == -1 {
+				nextCluster = textLen
+			}
+			switch dir {
+			case di.FromTopLeft:
+				runesInCluster = nextCluster - currentCluster
+			case di.TowardTopLeft:
+				runesInCluster = previousCluster - currentCluster
+			}
+			previousCluster = g
+		}
+		glyphs[i].GlyphCount = glyphsInCluster
+		glyphs[i].RuneCount = runesInCluster
+	}
+}
+
 func fixed266ToFloat(i fixed.Int26_6) float32 {
 	return float32(float64(i) / 64)
 }

render_image.go

@@ -193,12 +193,12 @@ func drawText(paragraphs []string, width, height int) image.Image {
 		for _, line := range lines {
 			for _, out := range line {
 				charsWritten, _ := drawShapedRunAt(
+					img,
 					FONT_SIZE,
 					color,
 					out,
 					emojiMask,
 					totalCharsWritten,
-					img,
 					0,
 					FONT_SIZE*lineNumber*12/10,
 				)

samples.txt

@@ -11,3 +11,5 @@ nevent1qqsy05v33j3w6klkfhy6taud0d7g7n6e7z9mt0z20aenqfam3lzywrcpzpmhxue69uhnzdps9
 nevent1qqszl72lntw6qdx2dc0fet9yrjpxlvh98ww7w3egm5ey5h0dwwzjg2gpzpmhxue69uhnzdps9enrw73wd9hsygyxl5wgpsraaw7r6xffxaajf49lvk594ung4u2umg4veez5manshc4w8pdp
 nevent1qqsyxahlr82z786vyhg0u2ycx46wrwwz5tap66udeffxle7phduthmspzpmhxue69uhnzdps9enrw73wd9hsygpcrk7vwyuw4wd8r6q5c4ur0jwky06qxmkqys80xq3nq6z0lj9n3u9zd3r4
 nevent1qqstj4gq2q9utuazaagzx0cd0n3jlm97p2084rfrvw2f8ytfa8ec2qgpp4mhxue69uhhjctzw5hx6egzypmezxyxju0727078e8epkwu06gle46g2rwzs5mgrll3we2tyxqfzuwew5q
+nevent1qqs9jhjm7cy3gtwlw3zvgrcras8nkvjs8gz0cvnmepmed4k92z8uxgspzpmhxue69uhkummnw3ezuamfdejsygy0mgsen3pen7khlaqjddqz7867u9tlm0jvp9g7vn07xx4wepml7yux7e7e
+nevent1qqsq9kxxrsmck2sj5gdu04llcjku5ex65eqyu5ec4a9nyr4ymudezegpp4mhxue69uhkummn9ekx7mqppamhxue69uhkummnw3ezumt0d5q3gamnwvaz7tmwdaehgu3wdau8gu3wv3jhvqgcwaehxw309ac82cnvd93juun9d3shj6twvuhxjmcpz3mhxue69uhhyetvv9ujumn0wd68ytnzvuq3samnwvaz7tmjv4kxz7fwdehhxamgv4ex2tnrdaksygqzm86kwmllcvull62dl2ech6lzrns303h32zwejg4g94z57ssd5gukzwe4