Austin/mihomo
1
0
Fork 0
mirror of https://github.com/MetaCubeX/mihomo.git synced 2024-11-16 03:32:33 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

chore: support ETag for update geo

Browse Source
This commit is contained in:
wwqgtxx 2024-09-22 13:57:57 +08:00
parent 5d242510c8
commit b7cb6774bf
11 changed files with 117 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
adapter/provider/parser.go
View File

@ -111,7 +111,7 @@ func ParseProxyProvider(name string, mapping map[string]any) (types.ProxyProvide
return nil, fmt.Errorf("%w: %s", errSubPath, path)
}
}
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header)
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, schema.Header, resource.DefaultHttpTimeout)
default:
return nil, fmt.Errorf("%w: %s", errVehicleType, schema.Type)
}

20
component/resource/fetcher.go
View File

@ -3,7 +3,6 @@ package resource
import (
"context"
"os"
"path/filepath"
"time"
types "github.com/metacubex/mihomo/constant/provider"
@ -13,11 +12,6 @@ import (
"github.com/samber/lo"
)
var (
fileMode os.FileMode = 0o666
dirMode os.FileMode = 0o755
)
type Parser[V any] func([]byte) (V, error)
type Fetcher[V any] struct {
@ -118,7 +112,7 @@ func (f *Fetcher[V]) loadBuf(buf []byte, hash types.HashType, updateFile bool) (
}
if updateFile {
if err = safeWrite(f.vehicle.Path(), buf); err != nil {
if err = f.vehicle.Write(buf); err != nil {
return lo.Empty[V](), false, err
}
}
@ -205,18 +199,6 @@ func (f *Fetcher[V]) updateWithLog() {
return
}
func safeWrite(path string, buf []byte) error {
dir := filepath.Dir(path)
if _, err := os.Stat(dir); os.IsNotExist(err) {
if err := os.MkdirAll(dir, dirMode); err != nil {
return err
}
}
return os.WriteFile(path, buf, fileMode)
}
func NewFetcher[V any](name string, interval time.Duration, vehicle types.Vehicle, parser Parser[V], onUpdate func(V)) *Fetcher[V] {
ctx, cancel := context.WithCancel(context.Background())
return &Fetcher[V]{

50
component/resource/vehicle.go
View File

@ -6,6 +6,7 @@ import (
"io"
"net/http"
"os"
"path/filepath"
"time"
mihomoHttp "github.com/metacubex/mihomo/component/http"
@ -13,6 +14,25 @@ import (
types "github.com/metacubex/mihomo/constant/provider"
)
const (
DefaultHttpTimeout = time.Second * 20
fileMode os.FileMode = 0o666
dirMode os.FileMode = 0o755
)
func safeWrite(path string, buf []byte) error {
dir := filepath.Dir(path)
if _, err := os.Stat(dir); os.IsNotExist(err) {
if err := os.MkdirAll(dir, dirMode); err != nil {
return err
}
}
return os.WriteFile(path, buf, fileMode)
}
type FileVehicle struct {
path string
}
@ -42,15 +62,20 @@ func (f *FileVehicle) Proxy() string {
return ""
}
func (f *FileVehicle) Write(buf []byte) error {
return safeWrite(f.path, buf)
}
func NewFileVehicle(path string) *FileVehicle {
return &FileVehicle{path: path}
}
type HTTPVehicle struct {
url string
path string
proxy string
header http.Header
url string
path string
proxy string
header http.Header
timeout time.Duration
}
func (h *HTTPVehicle) Url() string {
@ -69,8 +94,12 @@ func (h *HTTPVehicle) Proxy() string {
return h.proxy
}
func (h *HTTPVehicle) Write(buf []byte) error {
return safeWrite(h.path, buf)
}
func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []byte, hash types.HashType, err error) {
ctx, cancel := context.WithTimeout(ctx, time.Second*20)
ctx, cancel := context.WithTimeout(ctx, h.timeout)
defer cancel()
header := h.header
setIfNoneMatch := false
@ -107,11 +136,12 @@ func (h *HTTPVehicle) Read(ctx context.Context, oldHash types.HashType) (buf []b
return
}
func NewHTTPVehicle(url string, path string, proxy string, header http.Header) *HTTPVehicle {
func NewHTTPVehicle(url string, path string, proxy string, header http.Header, timeout time.Duration) *HTTPVehicle {
return &HTTPVehicle{
url: url,
path: path,
proxy: proxy,
header: header,
url: url,
path: path,
proxy: proxy,
header: header,
timeout: timeout,
}
}

75
component/updater/update_geo.go
View File

@ -13,7 +13,9 @@ import (
"github.com/metacubex/mihomo/component/geodata"
_ "github.com/metacubex/mihomo/component/geodata/standard"
"github.com/metacubex/mihomo/component/mmdb"
"github.com/metacubex/mihomo/component/resource"
C "github.com/metacubex/mihomo/constant"
P "github.com/metacubex/mihomo/constant/provider"
"github.com/metacubex/mihomo/log"
"github.com/oschwald/maxminddb-golang"
@ -43,30 +45,52 @@ func SetGeoUpdateInterval(newGeoUpdateInterval int) {
}
func UpdateMMDB() (err error) {
defer mmdb.ReloadIP()
data, err := downloadForBytes(geodata.MmdbUrl())
vehicle := resource.NewHTTPVehicle(geodata.MmdbUrl(), C.Path.MMDB(), "", nil, defaultHttpTimeout)
var oldHash P.HashType
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
oldHash = P.MakeHash(buf)
}
data, hash, err := vehicle.Read(context.Background(), oldHash)
if err != nil {
return fmt.Errorf("can't download MMDB database file: %w", err)
}
if oldHash.Equal(hash) { // same hash, ignored
return nil
}
if len(data) == 0 {
return fmt.Errorf("can't download MMDB database file: no data")
}
instance, err := maxminddb.FromBytes(data)
if err != nil {
return fmt.Errorf("invalid MMDB database file: %s", err)
}
_ = instance.Close()
defer mmdb.ReloadIP()
mmdb.IPInstance().Reader.Close() // mmdb is loaded with mmap, so it needs to be closed before overwriting the file
if err = saveFile(data, C.Path.MMDB()); err != nil {
if err = vehicle.Write(data); err != nil {
return fmt.Errorf("can't save MMDB database file: %w", err)
}
return nil
}
func UpdateASN() (err error) {
defer mmdb.ReloadASN()
data, err := downloadForBytes(geodata.ASNUrl())
vehicle := resource.NewHTTPVehicle(geodata.ASNUrl(), C.Path.ASN(), "", nil, defaultHttpTimeout)
var oldHash P.HashType
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
oldHash = P.MakeHash(buf)
}
data, hash, err := vehicle.Read(context.Background(), oldHash)
if err != nil {
return fmt.Errorf("can't download ASN database file: %w", err)
}
if oldHash.Equal(hash) { // same hash, ignored
return nil
}
if len(data) == 0 {
return fmt.Errorf("can't download ASN database file: no data")
}
instance, err := maxminddb.FromBytes(data)
if err != nil {
@ -74,42 +98,69 @@ func UpdateASN() (err error) {
}
_ = instance.Close()
defer mmdb.ReloadASN()
mmdb.ASNInstance().Reader.Close() // mmdb is loaded with mmap, so it needs to be closed before overwriting the file
if err = saveFile(data, C.Path.ASN()); err != nil {
if err = vehicle.Write(data); err != nil {
return fmt.Errorf("can't save ASN database file: %w", err)
}
return nil
}
func UpdateGeoIp() (err error) {
defer geodata.ClearGeoIPCache()
geoLoader, err := geodata.GetGeoDataLoader("standard")
data, err := downloadForBytes(geodata.GeoIpUrl())
vehicle := resource.NewHTTPVehicle(geodata.GeoIpUrl(), C.Path.GeoIP(), "", nil, defaultHttpTimeout)
var oldHash P.HashType
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
oldHash = P.MakeHash(buf)
}
data, hash, err := vehicle.Read(context.Background(), oldHash)
if err != nil {
return fmt.Errorf("can't download GeoIP database file: %w", err)
}
if oldHash.Equal(hash) { // same hash, ignored
return nil
}
if len(data) == 0 {
return fmt.Errorf("can't download GeoIP database file: no data")
}
if _, err = geoLoader.LoadIPByBytes(data, "cn"); err != nil {
return fmt.Errorf("invalid GeoIP database file: %s", err)
}
if err = saveFile(data, C.Path.GeoIP()); err != nil {
defer geodata.ClearGeoIPCache()
if err = vehicle.Write(data); err != nil {
return fmt.Errorf("can't save GeoIP database file: %w", err)
}
return nil
}
func UpdateGeoSite() (err error) {
defer geodata.ClearGeoSiteCache()
geoLoader, err := geodata.GetGeoDataLoader("standard")
data, err := downloadForBytes(geodata.GeoSiteUrl())
vehicle := resource.NewHTTPVehicle(geodata.GeoSiteUrl(), C.Path.GeoSite(), "", nil, defaultHttpTimeout)
var oldHash P.HashType
if buf, err := os.ReadFile(vehicle.Path()); err == nil {
oldHash = P.MakeHash(buf)
}
data, hash, err := vehicle.Read(context.Background(), oldHash)
if err != nil {
return fmt.Errorf("can't download GeoSite database file: %w", err)
}
if oldHash.Equal(hash) { // same hash, ignored
return nil
}
if len(data) == 0 {
return fmt.Errorf("can't download GeoSite database file: no data")
}
if _, err = geoLoader.LoadSiteByBytes(data, "cn"); err != nil {
return fmt.Errorf("invalid GeoSite database file: %s", err)
}
if err = saveFile(data, C.Path.GeoSite()); err != nil {
defer geodata.ClearGeoSiteCache()
if err = vehicle.Write(data); err != nil {
return fmt.Errorf("can't save GeoSite database file: %w", err)
}
return nil

4
component/updater/update_ui.go
View File

@ -17,12 +17,12 @@ import (
var (
ExternalUIURL string
ExternalUIPath string
AutoUpdateUI bool
AutoDownloadUI bool
)
var xdMutex sync.Mutex
func UpdateUI() error {
func DownloadUI() error {
xdMutex.Lock()
defer xdMutex.Unlock()

4
component/updater/utils.go
View File

@ -13,8 +13,10 @@ import (
"golang.org/x/exp/constraints"
)
const defaultHttpTimeout = time.Second * 90
func downloadForBytes(url string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*90)
ctx, cancel := context.WithTimeout(context.Background(), defaultHttpTimeout)
defer cancel()
resp, err := mihomoHttp.HttpRequest(ctx, url, http.MethodGet, nil, nil)
if err != nil {

4
config/config.go
View File

@ -701,7 +701,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) {
// checkout externalUI exist
if cfg.ExternalUI != "" {
updater.AutoUpdateUI = true
updater.AutoDownloadUI = true
updater.ExternalUIPath = C.Path.Resolve(cfg.ExternalUI)
} else {
// default externalUI path
@ -710,7 +710,7 @@ func parseGeneral(cfg *RawConfig) (*General, error) {
// checkout UIpath/name exist
if cfg.ExternalUIName != "" {
updater.AutoUpdateUI = true
updater.AutoDownloadUI = true
updater.ExternalUIPath = path.Join(updater.ExternalUIPath, cfg.ExternalUIName)
}

1
constant/provider/interface.go
View File

@ -33,6 +33,7 @@ func (v VehicleType) String() string {
type Vehicle interface {
Read(ctx context.Context, oldHash HashType) (buf []byte, hash HashType, err error)
Write(buf []byte) error
Path() string
Url() string
Proxy() string

4
hub/executor/executor.go
View File

@ -381,13 +381,13 @@ func updateTunnels(tunnels []LC.Tunnel) {
}
func initExternalUI() {
if updater.AutoUpdateUI {
if updater.AutoDownloadUI {
dirEntries, _ := os.ReadDir(updater.ExternalUIPath)
if len(dirEntries) > 0 {
log.Infoln("UI already exists, skip downloading")
} else {
log.Infoln("External UI downloading ...")
updater.UpdateUI()
updater.DownloadUI()
}
}
}

2
hub/route/upgrade.go
View File

@ -47,7 +47,7 @@ func upgradeCore(w http.ResponseWriter, r *http.Request) {
}
func updateUI(w http.ResponseWriter, r *http.Request) {
err := updater.UpdateUI()
err := updater.DownloadUI()
if err != nil {
log.Warnln("%s", err)
render.Status(r, http.StatusInternalServerError)

2
rules/provider/parse.go
View File

@ -53,7 +53,7 @@ func ParseRuleProvider(name string, mapping map[string]interface{}, parse func(t
return nil, fmt.Errorf("%w: %s", errSubPath, path)
}
}
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil)
vehicle = resource.NewHTTPVehicle(schema.URL, path, schema.Proxy, nil, resource.DefaultHttpTimeout)
default:
return nil, fmt.Errorf("unsupported vehicle type: %s", schema.Type)
}