mirror of
https://github.com/sajari/docconv.git
synced 2026-10-16 21:21:59 +08:00
David Howden
32 lines
635 B
Go
32 lines
635 B
Go
package docconv
|
|
|
|
import (
|
|
"bytes"
|
|
"io"
|
|
|
|
"github.com/advancedlogic/GoOse"
|
|
)
|
|
|
|
// ConvertURL fetches the HTML page at the URL given in the io.Reader.
|
|
func ConvertURL(input io.Reader, readability bool) (string, map[string]string, error) {
|
|
meta := make(map[string]string)
|
|
|
|
buf := new(bytes.Buffer)
|
|
_, err := buf.ReadFrom(input)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
g := goose.New()
|
|
article, err := g.ExtractFromURL(buf.String())
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
|
|
meta["title"] = article.Title
|
|
meta["description"] = article.MetaDescription
|
|
meta["image"] = article.TopImage
|
|
|
|
return article.CleanedText, meta, nil
|
|
}
|