post inicial

2026-09-27 16:04:02 -06:00
parent d84587e1f3
commit 52864794f5
67 changed files with 15023 additions and 2 deletions
@@ -0,0 +1,89 @@
+package tools
+
+import (
+	"html"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// htmlPage es una página convertida a texto legible para el modelo.
+type htmlPage struct {
+	Title string
+	Text  string
+	Links []string // URLs absolutas, en orden de aparición; [n] en el texto
+}
+
+var (
+	dropBlocksRe = regexp.MustCompile(`(?is)<(script|style|noscript|svg|head|template|iframe)\b.*?</\s*(script|style|noscript|svg|head|template|iframe)\s*>`)
+	commentRe    = regexp.MustCompile(`(?s)<!--.*?-->`)
+	titleRe      = regexp.MustCompile(`(?is)<title[^>]*>(.*?)</title>`)
+	anchorRe     = regexp.MustCompile(`(?is)<a\b[^>]*?\bhref\s*=\s*("([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>(.*?)</a\s*>`)
+	blockTagRe   = regexp.MustCompile(`(?i)</?(p|div|br|tr|table|section|article|header|footer|nav|main|aside|h[1-6]|ul|ol|dl|dt|dd|pre|blockquote|hr|form)\b[^>]*>`)
+	liTagRe      = regexp.MustCompile(`(?i)<li\b[^>]*>`)
+	cellOpenRe   = regexp.MustCompile(`(?i)<(td|th)\b[^>]*>`)
+	cellCloseRe  = regexp.MustCompile(`(?i)</(td|th)\s*>`)
+	anyTagRe     = regexp.MustCompile(`(?s)<[^>]+>`)
+	spacesRe     = regexp.MustCompile(`[ \t\f\v\x{00a0}]+`)
+	blankLinesRe = regexp.MustCompile(`\n[ \t]*(\n[ \t]*)+`)
+)
+
+// htmlToText convierte HTML en texto plano conservando la estructura básica
+// (párrafos, listas, tablas) y numerando los enlaces para poder seguirlos.
+//
+// No pretende ser un navegador: su objetivo es que documentación, avisos de
+// seguridad y páginas de issues sean legibles para el modelo con el mínimo
+// de tokens. Se implementa con la biblioteca estándar para no añadir
+// dependencias.
+func htmlToText(src, base string) htmlPage {
+	var p htmlPage
+	if m := titleRe.FindStringSubmatch(src); m != nil {
+		p.Title = strings.TrimSpace(html.UnescapeString(anyTagRe.ReplaceAllString(m[1], "")))
+	}
+	s := commentRe.ReplaceAllString(src, "")
+	s = dropBlocksRe.ReplaceAllString(s, " ")
+
+	baseURL, _ := url.Parse(base)
+	seen := map[string]int{}
+	s = anchorRe.ReplaceAllStringFunc(s, func(a string) string {
+		m := anchorRe.FindStringSubmatch(a)
+		href := html.UnescapeString(m[2] + m[3] + m[4])
+		label := strings.TrimSpace(anyTagRe.ReplaceAllString(m[5], ""))
+		if strings.HasPrefix(href, "#") || strings.HasPrefix(strings.ToLower(href), "javascript:") || strings.HasPrefix(strings.ToLower(href), "mailto:") {
+			return label
+		}
+		if baseURL != nil {
+			if u, err := baseURL.Parse(href); err == nil {
+				href = u.String()
+			}
+		}
+		n, ok := seen[href]
+		if !ok {
+			p.Links = append(p.Links, href)
+			n = len(p.Links)
+			seen[href] = n
+		}
+		if label == "" {
+			return ""
+		}
+		return label + " [" + strconv.Itoa(n) + "]"
+	})
+
+	s = liTagRe.ReplaceAllString(s, "\n- ")
+	s = cellOpenRe.ReplaceAllString(s, "")
+	s = cellCloseRe.ReplaceAllString(s, " | ")
+	s = blockTagRe.ReplaceAllString(s, "\n")
+	s = anyTagRe.ReplaceAllString(s, "")
+	s = html.UnescapeString(s)
+	s = spacesRe.ReplaceAllString(s, " ")
+	lines := strings.Split(s, "\n")
+	for i, l := range lines {
+		lines[i] = strings.TrimSuffix(strings.TrimSpace(l), " |")
+		lines[i] = strings.TrimSpace(strings.TrimSuffix(lines[i], "|"))
+	}
+	s = strings.Join(lines, "\n")
+	s = blankLinesRe.ReplaceAllString(s, "\n\n")
+	p.Text = strings.TrimSpace(s)
+	return p
+}