mirror of
https://github.com/usememos/memos.git
synced 2026-11-19 10:29:11 +01:00
261 lines
6.3 KiB
Go
261 lines
6.3 KiB
Go
package parser
|
|
|
|
import (
|
|
"unicode/utf8"
|
|
|
|
"github.com/yuin/goldmark/util"
|
|
)
|
|
|
|
//go:generate go run ./tagdata -output tag_unicode_tables.go
|
|
|
|
type codePointRange struct {
|
|
lo rune
|
|
hi rune
|
|
}
|
|
|
|
// TagMatch is one lexical tag match within an eligible literal-source run.
|
|
type TagMatch struct {
|
|
// Start is the byte offset of the introducer within the source run.
|
|
Start int
|
|
// End is the exclusive byte offset of the recognized source spelling.
|
|
End int
|
|
// Value is the emitted direct tag value without the introducer.
|
|
Value []byte
|
|
}
|
|
|
|
func containsCodePoint(ranges []codePointRange, r rune) bool {
|
|
low, high := 0, len(ranges)
|
|
for low < high {
|
|
middle := low + (high-low)/2
|
|
current := ranges[middle]
|
|
switch {
|
|
case r < current.lo:
|
|
high = middle
|
|
case r > current.hi:
|
|
low = middle + 1
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func isXIDContinue(r rune) bool {
|
|
return containsCodePoint(unicode17XIDContinue[:], r)
|
|
}
|
|
|
|
func isDefaultIgnorable(r rune) bool {
|
|
return containsCodePoint(unicode17DefaultIgnorable[:], r)
|
|
}
|
|
|
|
func isCombiningMark(r rune) bool {
|
|
return containsCodePoint(unicode17CombiningMarks[:], r)
|
|
}
|
|
|
|
func isSegmentStarter(r rune) bool {
|
|
return r == '-' || r == '+' || r == '&' || isXIDContinue(r) && !isCombiningMark(r) && !isDefaultIgnorable(r)
|
|
}
|
|
|
|
func isSegmentContinuation(r rune) bool {
|
|
return r == '-' || r == '+' || r == '&' || isXIDContinue(r) && !isDefaultIgnorable(r)
|
|
}
|
|
|
|
func isApostropheJoiner(r rune) bool {
|
|
return r == '\'' || r == '\u2019'
|
|
}
|
|
|
|
func startsNonCombiningXIDContinuation(source []byte) bool {
|
|
if len(source) == 0 || matchFullyQualifiedEmoji(source) > 0 {
|
|
return false
|
|
}
|
|
r, size := utf8.DecodeRune(source)
|
|
if r == utf8.RuneError && size == 1 {
|
|
return false
|
|
}
|
|
return isXIDContinue(r) && !isCombiningMark(r) && !isDefaultIgnorable(r)
|
|
}
|
|
|
|
func matchFullyQualifiedEmoji(source []byte) int {
|
|
limit := min(len(source), maxEmoji17SequenceBytes)
|
|
match := 0
|
|
for pos := 0; pos < limit; {
|
|
r, size := utf8.DecodeRune(source[pos:])
|
|
if r == utf8.RuneError && size == 1 {
|
|
break
|
|
}
|
|
pos += size
|
|
if pos > limit {
|
|
break
|
|
}
|
|
if _, ok := emoji17FullyQualified[string(source[:pos])]; ok {
|
|
match = pos
|
|
}
|
|
}
|
|
return match
|
|
}
|
|
|
|
func scanTagSegment(source []byte) (int, []byte, bool) {
|
|
pos := 0
|
|
var value []byte
|
|
previousWasXIDContinuation := false
|
|
|
|
starter:
|
|
for pos < len(source) {
|
|
if source[pos] == '&' && characterReferenceLength(source[pos:]) > 0 {
|
|
return 0, nil, false
|
|
}
|
|
if emojiLength := matchFullyQualifiedEmoji(source[pos:]); emojiLength > 0 {
|
|
value = append(value, source[pos:pos+emojiLength]...)
|
|
pos += emojiLength
|
|
previousWasXIDContinuation = false
|
|
break
|
|
}
|
|
r, size := utf8.DecodeRune(source[pos:])
|
|
if r == utf8.RuneError && size == 1 {
|
|
return 0, nil, false
|
|
}
|
|
switch {
|
|
case isDefaultIgnorable(r):
|
|
pos += size
|
|
case isXIDContinue(r) && isCombiningMark(r):
|
|
pos += size
|
|
case isSegmentStarter(r):
|
|
value = append(value, source[pos:pos+size]...)
|
|
pos += size
|
|
previousWasXIDContinuation = isXIDContinue(r) && !isDefaultIgnorable(r)
|
|
break starter
|
|
default:
|
|
return 0, nil, false
|
|
}
|
|
}
|
|
|
|
if len(value) == 0 {
|
|
return 0, nil, false
|
|
}
|
|
|
|
for pos < len(source) {
|
|
if source[pos] == '&' && characterReferenceLength(source[pos:]) > 0 {
|
|
break
|
|
}
|
|
if emojiLength := matchFullyQualifiedEmoji(source[pos:]); emojiLength > 0 {
|
|
value = append(value, source[pos:pos+emojiLength]...)
|
|
pos += emojiLength
|
|
previousWasXIDContinuation = false
|
|
continue
|
|
}
|
|
r, size := utf8.DecodeRune(source[pos:])
|
|
if r == utf8.RuneError && size == 1 {
|
|
break
|
|
}
|
|
switch {
|
|
case isDefaultIgnorable(r):
|
|
pos += size
|
|
previousWasXIDContinuation = false
|
|
case isApostropheJoiner(r) && previousWasXIDContinuation && startsNonCombiningXIDContinuation(source[pos+size:]):
|
|
value = append(value, source[pos:pos+size]...)
|
|
pos += size
|
|
previousWasXIDContinuation = false
|
|
case isSegmentContinuation(r):
|
|
value = append(value, source[pos:pos+size]...)
|
|
pos += size
|
|
previousWasXIDContinuation = isXIDContinue(r) && !isDefaultIgnorable(r)
|
|
default:
|
|
return pos, value, true
|
|
}
|
|
}
|
|
|
|
return pos, value, true
|
|
}
|
|
|
|
func scanTagIdentifier(source []byte) (int, []byte, bool) {
|
|
consumed, value, ok := scanTagSegment(source)
|
|
if !ok {
|
|
return 0, nil, false
|
|
}
|
|
|
|
for consumed < len(source) && source[consumed] == '/' {
|
|
segmentLength, segmentValue, segmentOK := scanTagSegment(source[consumed+1:])
|
|
if !segmentOK {
|
|
break
|
|
}
|
|
value = append(value, '/')
|
|
value = append(value, segmentValue...)
|
|
consumed += 1 + segmentLength
|
|
}
|
|
|
|
return consumed, value, true
|
|
}
|
|
|
|
func characterReferenceLength(source []byte) int {
|
|
if len(source) < 3 || source[0] != '&' {
|
|
return 0
|
|
}
|
|
|
|
if source[1] == '#' {
|
|
pos, maxDigits := 2, 7
|
|
isDigit := func(value byte) bool { return value >= '0' && value <= '9' }
|
|
if pos < len(source) && (source[pos] == 'x' || source[pos] == 'X') {
|
|
pos++
|
|
maxDigits = 6
|
|
isDigit = func(value byte) bool {
|
|
return value >= '0' && value <= '9' || value >= 'a' && value <= 'f' || value >= 'A' && value <= 'F'
|
|
}
|
|
}
|
|
start := pos
|
|
for pos < len(source) && pos-start < maxDigits && isDigit(source[pos]) {
|
|
pos++
|
|
}
|
|
if pos > start && pos < len(source) && source[pos] == ';' {
|
|
return pos + 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
pos := 1
|
|
for pos < len(source) && pos <= 32 && util.IsAlphaNumeric(source[pos]) {
|
|
pos++
|
|
}
|
|
if pos == 1 || pos >= len(source) || source[pos] != ';' {
|
|
return 0
|
|
}
|
|
if _, ok := util.LookUpHTML5EntityByName(string(source[1:pos])); !ok {
|
|
return 0
|
|
}
|
|
return pos + 1
|
|
}
|
|
|
|
// FindTagMatches enumerates tag candidates in one eligible
|
|
// literal-source run. Markdown structure must be resolved before calling it.
|
|
func FindTagMatches(source []byte) []TagMatch {
|
|
var matches []TagMatch
|
|
for pos := 0; pos < len(source); {
|
|
if emojiLength := matchFullyQualifiedEmoji(source[pos:]); emojiLength > 0 {
|
|
pos += emojiLength
|
|
continue
|
|
}
|
|
if referenceLength := characterReferenceLength(source[pos:]); referenceLength > 0 {
|
|
pos += referenceLength
|
|
continue
|
|
}
|
|
if source[pos] == '\\' && pos+1 < len(source) && util.IsPunct(source[pos+1]) {
|
|
pos += 2
|
|
continue
|
|
}
|
|
if source[pos] != '#' {
|
|
_, size := utf8.DecodeRune(source[pos:])
|
|
pos += size
|
|
continue
|
|
}
|
|
|
|
consumed, value, ok := scanTagIdentifier(source[pos+1:])
|
|
if !ok {
|
|
pos++
|
|
continue
|
|
}
|
|
end := pos + consumed + 1
|
|
matches = append(matches, TagMatch{Start: pos, End: end, Value: value})
|
|
pos = end
|
|
}
|
|
return matches
|
|
}
|