Merge pull request #48 from xushiwei/q

hdq: http/cached

chore/pysigfetch/pysigfetch.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/goplus/hdq/fetcher"
 	_ "github.com/goplus/hdq/fetcher/torch"
+	_ "github.com/goplus/hdq/stream/http/cached"
 )
 
 type module struct {

hdq.go

@@ -23,7 +23,6 @@ import (
 	"github.com/goplus/hdq/stream"
 	"golang.org/x/net/html"
 
-	_ "github.com/goplus/hdq/stream/http"
 	_ "github.com/goplus/hdq/stream/zip"
 )
 

stream/http/cached/cached.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cached
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/url"
+	"os"
+	"path"
+
+	"github.com/goplus/hdq/stream"
+	"github.com/goplus/hdq/stream/http"
+)
+
+// -------------------------------------------------------------------------------------
+
+var (
+	cacheDir = getCacheDir()
+)
+
+func getCacheDir() string {
+	root, err := os.UserCacheDir()
+	if err != nil {
+		panic(err)
+	}
+	dir := root + "/hdq/http/"
+	os.MkdirAll(dir, 0755)
+	return dir
+}
+
+// -------------------------------------------------------------------------------------
+
+// TODO(xsw): add checksum to cache file
+func WriteCache(cacheFile string, url string) (err error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	f, err := os.Create(cacheFile)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+	_, err = io.Copy(f, resp.Body)
+	return
+}
+
+func ReadCache(cacheFile string, fi fs.FileInfo) (ret io.ReadCloser, err error) {
+	return os.Open(cacheFile)
+}
+
+// -------------------------------------------------------------------------------------
+
+// Open opens a http file object.
+func Open(url_ string) (ret io.ReadCloser, err error) {
+	u, err := url.Parse(url_)
+	if err != nil {
+		return
+	}
+	fname := path.Base(u.Path)
+	ext := path.Ext(fname)
+	hash := md5.Sum([]byte(url_))
+	hashstr := base64.RawURLEncoding.EncodeToString(hash[:])
+	fname = fmt.Sprintf("%s-%s%s", fname[:len(fname)-len(ext)], hashstr, ext)
+	file := cacheDir + fname
+	if fi, e := os.Stat(file); e == nil {
+		if ret, err = ReadCache(file, fi); err == nil { // cache hit
+			return
+		}
+	}
+	if err = WriteCache(file, url_); err != nil {
+		return // write cache failed
+	}
+	return ReadCache(file, nil)
+}
+
+func init() {
+	stream.Register("http", Open)
+	stream.Register("https", Open)
+}
+
+// -------------------------------------------------------------------------------------

stream/http/httpstrm.go

@@ -18,8 +18,6 @@ package http
 import (
 	"io"
 	"net/http"
-
-	"github.com/goplus/hdq/stream"
 )
 
 var (
@@ -32,14 +30,14 @@ var (
 
 // Open opens a http file object.
 func Open(url string) (io.ReadCloser, error) {
-	resp, err := httpGet(url)
+	resp, err := Get(url)
 	if err != nil {
 		return nil, err
 	}
 	return resp.Body, nil
 }
 
-func httpGet(url string) (resp *http.Response, err error) {
+func Get(url string) (resp *http.Response, err error) {
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return nil, err
@@ -53,9 +51,4 @@ func httpGet(url string) (resp *http.Response, err error) {
 	return http.DefaultClient.Do(req)
 }
 
-func init() {
-	stream.Register("http", Open)
-	stream.Register("https", Open)
-}
-
 // -------------------------------------------------------------------------------------

stream/http/nocache/nocache.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2024 The GoPlus Authors (goplus.org)
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nocache
+
+import (
+	"github.com/goplus/hdq/stream"
+	"github.com/goplus/hdq/stream/http"
+)
+
+func init() {
+	stream.Register("http", http.Open)
+	stream.Register("https", http.Open)
+}