|
|
|
@ -1,6 +1,7 @@
|
|
|
|
package httpgetter
|
|
|
|
package httpgetter
|
|
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
import (
|
|
|
|
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"io"
|
|
|
|
"net"
|
|
|
|
"net"
|
|
|
|
"net/http"
|
|
|
|
"net/http"
|
|
|
|
@ -53,6 +54,7 @@ func GetHTMLMeta(urlStr string) (*HTMLMeta, error) {
|
|
|
|
// TODO: limit the size of the response body
|
|
|
|
// TODO: limit the size of the response body
|
|
|
|
|
|
|
|
|
|
|
|
htmlMeta := extractHTMLMeta(response.Body)
|
|
|
|
htmlMeta := extractHTMLMeta(response.Body)
|
|
|
|
|
|
|
|
enrichSiteMeta(response.Request.URL, htmlMeta)
|
|
|
|
return htmlMeta, nil
|
|
|
|
return htmlMeta, nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
@ -151,3 +153,14 @@ func validateURL(urlStr string) error {
|
|
|
|
|
|
|
|
|
|
|
|
return nil
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
func enrichSiteMeta(url *url.URL, meta *HTMLMeta) {
|
|
|
|
|
|
|
|
if url.Hostname() == "www.youtube.com" {
|
|
|
|
|
|
|
|
if url.Path == "/watch" {
|
|
|
|
|
|
|
|
vid := url.Query().Get("v")
|
|
|
|
|
|
|
|
if vid != "" {
|
|
|
|
|
|
|
|
meta.Image = fmt.Sprintf("https://img.youtube.com/vi/%s/mqdefault.jpg", vid)
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|