feat(utils): 添加QQWry IP数据库解析功能及HTTP查询服务

2025-11-03 16:38:53 +00:00
parent f61e6cc937
commit ed4d961e32
8 changed files with 454 additions and 0 deletions
--- a/common/utils/qqwry/LICENSE
+++ b/common/utils/qqwry/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 xiaoqidun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
--- a/common/utils/qqwry/README.md
+++ b/common/utils/qqwry/README.md
@@ -0,0 +1,71 @@
+# QQWry [![Go Reference](https://pkg.go.dev/badge/github.com/xiaoqidun/qqwry.svg)](https://pkg.go.dev/github.com/xiaoqidun/qqwry)
+
+Golang QQWry，高性能纯真IP查询库。
+
+# 使用须知
+
+1. dat格式仅支持ipv4查询。
+2. ipdb格式支持ipv4和ipv6查询。
+
+# 使用说明
+
+```go
+package main
+
+import (
+	"fmt"
+	"github.com/xiaoqidun/qqwry"
+)
+
+func main() {
+	// 从文件加载IP数据库
+	if err := qqwry.LoadFile("qqwry.ipdb"); err != nil {
+		panic(err)
+	}
+	// 从内存或缓存查询IP
+	location, err := qqwry.QueryIP("119.29.29.29")
+	if err != nil {
+		fmt.Printf("错误：%v\n", err)
+		return
+	}
+	fmt.Printf("国家：%s，省份：%s，城市：%s，区县：%s，运营商：%s\n",
+		location.Country,
+		location.Province,
+		location.City,
+		location.District,
+		location.ISP,
+	)
+}
+```
+
+# IP数据库
+
+- DAT格式：[https://aite.xyz/share-file/qqwry/qqwry.dat](https://aite.xyz/share-file/qqwry/qqwry.dat)
+- IPDB格式：[https://aite.xyz/share-file/qqwry/qqwry.ipdb](https://aite.xyz/share-file/qqwry/qqwry.ipdb)
+
+# 编译说明
+
+1. 下载IP数据库并放置于assets目录中。
+2. client和server需要go1.16的内嵌资源特性。
+3. 作为库使用，请直接引包，并不需要go1.16+才能编译。
+
+# 数据更新
+
+- 由于qqwry.dat缺乏更新，官方czdb格式又难以获得和分发，建议使用ipdb格式。
+- 这里的ipdb格式指metowolf提供的官方czdb格式转换而来的ipdb格式（纯真格式原版）。
+
+# 服务接口
+
+1. 自行根据需要调整server下源码。
+2. 可以通过-listen参数指定http服务地址。
+3. json api：curl http://127.0.0.1/ip/119.29.29.29
+
+# 特别感谢
+
+- 感谢[纯真IP库](https://www.cz88.net/)一直坚持为大家提供免费IP数据库。
+- 感谢[yinheli](https://github.com/yinheli)的[qqwry](https://github.com/yinheli/qqwry)项目，为我提供纯真ip库解析算法参考。
+- 感谢[metowolf](https://github.com/metowolf)的[qqwry.ipdb](https://github.com/metowolf/qqwry.ipdb)项目，提供纯真czdb转ipdb数据库。
+
+# 授权说明
+
+使用本类库你唯一需要做的就是把LICENSE文件往你用到的项目中拷贝一份。
--- a/common/utils/qqwry/client/client.go
+++ b/common/utils/qqwry/client/client.go
@@ -0,0 +1,37 @@
+package main
+
+import (
+	"fmt"
+	"github.com/xiaoqidun/qqwry"
+	"github.com/xiaoqidun/qqwry/assets"
+	"os"
+)
+
+func init() {
+	qqwry.LoadData(assets.QQWryIpdb)
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		return
+	}
+	queryIp := os.Args[1]
+	location, err := qqwry.QueryIP(queryIp)
+	if err != nil {
+		fmt.Printf("错误：%v\n", err)
+		return
+	}
+	emptyVal := func(val string) string {
+		if val != "" {
+			return val
+		}
+		return "未知"
+	}
+	fmt.Printf("国家：%s，省份：%s，城市：%s，区县：%s，运营商：%s\n",
+		emptyVal(location.Country),
+		emptyVal(location.Province),
+		emptyVal(location.City),
+		emptyVal(location.District),
+		emptyVal(location.ISP),
+	)
+}
--- a/common/utils/qqwry/go.mod
+++ b/common/utils/qqwry/go.mod
@@ -0,0 +1,8 @@
+module github.com/xiaoqidun/qqwry
+
+go 1.20
+
+require (
+	github.com/ipipdotnet/ipdb-go v1.3.3
+	golang.org/x/text v0.8.0
+)
--- a/common/utils/qqwry/go.sum
+++ b/common/utils/qqwry/go.sum
@@ -0,0 +1,4 @@
+github.com/ipipdotnet/ipdb-go v1.3.3 h1:GLSAW9ypLUd6EF9QNK2Uhxew9Jzs4XMJ9gOZEFnJm7U=
+github.com/ipipdotnet/ipdb-go v1.3.3/go.mod h1:yZ+8puwe3R37a/3qRftXo40nZVQbxYDLqls9o5foexs=
+golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
+golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
--- a/common/utils/qqwry/qqwry.go
+++ b/common/utils/qqwry/qqwry.go
@@ -0,0 +1,235 @@
+package qqwry
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"github.com/ipipdotnet/ipdb-go"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/transform"
+	"io"
+	"net"
+	"os"
+	"strings"
+	"sync"
+)
+
+var (
+	data          []byte
+	dataLen       uint32
+	ipdbCity      *ipdb.City
+	dataType      = dataTypeDat
+	locationCache = &sync.Map{}
+)
+
+const (
+	dataTypeDat  = 0
+	dataTypeIpdb = 1
+)
+
+const (
+	indexLen      = 7
+	redirectMode1 = 0x01
+	redirectMode2 = 0x02
+)
+
+type Location struct {
+	Country  string // 国家
+	Province string // 省份
+	City     string // 城市
+	District string // 区县
+	ISP      string // 运营商
+	IP       string // IP地址
+}
+
+func byte3ToUInt32(data []byte) uint32 {
+	i := uint32(data[0]) & 0xff
+	i |= (uint32(data[1]) << 8) & 0xff00
+	i |= (uint32(data[2]) << 16) & 0xff0000
+	return i
+}
+
+func gb18030Decode(src []byte) string {
+	in := bytes.NewReader(src)
+	out := transform.NewReader(in, simplifiedchinese.GB18030.NewDecoder())
+	d, _ := io.ReadAll(out)
+	return string(d)
+}
+
+// QueryIP 从内存或缓存查询IP
+func QueryIP(ip string) (location *Location, err error) {
+	if v, ok := locationCache.Load(ip); ok {
+		return v.(*Location), nil
+	}
+	switch dataType {
+	case dataTypeDat:
+		return QueryIPDat(ip)
+	case dataTypeIpdb:
+		return QueryIPIpdb(ip)
+	default:
+		return nil, errors.New("data type not support")
+	}
+}
+
+// QueryIPDat 从dat查询IP，仅加载dat格式数据库时使用
+func QueryIPDat(ipv4 string) (location *Location, err error) {
+	ip := net.ParseIP(ipv4).To4()
+	if ip == nil {
+		return nil, errors.New("ip is not ipv4")
+	}
+	ip32 := binary.BigEndian.Uint32(ip)
+	posA := binary.LittleEndian.Uint32(data[:4])
+	posZ := binary.LittleEndian.Uint32(data[4:8])
+	var offset uint32 = 0
+	for {
+		mid := posA + (((posZ-posA)/indexLen)>>1)*indexLen
+		buf := data[mid : mid+indexLen]
+		_ip := binary.LittleEndian.Uint32(buf[:4])
+		if posZ-posA == indexLen {
+			offset = byte3ToUInt32(buf[4:])
+			buf = data[mid+indexLen : mid+indexLen+indexLen]
+			if ip32 < binary.LittleEndian.Uint32(buf[:4]) {
+				break
+			} else {
+				offset = 0
+				break
+			}
+		}
+		if _ip > ip32 {
+			posZ = mid
+		} else if _ip < ip32 {
+			posA = mid
+		} else if _ip == ip32 {
+			offset = byte3ToUInt32(buf[4:])
+			break
+		}
+	}
+	if offset <= 0 {
+		return nil, errors.New("ip not found")
+	}
+	posM := offset + 4
+	mode := data[posM]
+	var ispPos uint32
+	var addr, isp string
+	switch mode {
+	case redirectMode1:
+		posC := byte3ToUInt32(data[posM+1 : posM+4])
+		mode = data[posC]
+		posCA := posC
+		if mode == redirectMode2 {
+			posCA = byte3ToUInt32(data[posC+1 : posC+4])
+			posC += 4
+		}
+		for i := posCA; i < dataLen; i++ {
+			if data[i] == 0 {
+				addr = string(data[posCA:i])
+				break
+			}
+		}
+		if mode != redirectMode2 {
+			posC += uint32(len(addr) + 1)
+		}
+		ispPos = posC
+	case redirectMode2:
+		posCA := byte3ToUInt32(data[posM+1 : posM+4])
+		for i := posCA; i < dataLen; i++ {
+			if data[i] == 0 {
+				addr = string(data[posCA:i])
+				break
+			}
+		}
+		ispPos = offset + 8
+	default:
+		posCA := offset + 4
+		for i := posCA; i < dataLen; i++ {
+			if data[i] == 0 {
+				addr = string(data[posCA:i])
+				break
+			}
+		}
+		ispPos = offset + uint32(5+len(addr))
+	}
+	if addr != "" {
+		addr = strings.TrimSpace(gb18030Decode([]byte(addr)))
+	}
+	ispMode := data[ispPos]
+	if ispMode == redirectMode1 || ispMode == redirectMode2 {
+		ispPos = byte3ToUInt32(data[ispPos+1 : ispPos+4])
+	}
+	if ispPos > 0 {
+		for i := ispPos; i < dataLen; i++ {
+			if data[i] == 0 {
+				isp = string(data[ispPos:i])
+				if isp != "" {
+					if strings.Contains(isp, "CZ88.NET") {
+						isp = ""
+					} else {
+						isp = strings.TrimSpace(gb18030Decode([]byte(isp)))
+					}
+				}
+				break
+			}
+		}
+	}
+	location = SplitResult(addr, isp, ipv4)
+	locationCache.Store(ipv4, location)
+	return location, nil
+}
+
+// QueryIPIpdb 从ipdb查询IP，仅加载ipdb格式数据库时使用
+func QueryIPIpdb(ip string) (location *Location, err error) {
+	ret, err := ipdbCity.Find(ip, "CN")
+	if err != nil {
+		return
+	}
+	location = SplitResult(ret[0], ret[1], ip)
+	locationCache.Store(ip, location)
+	return location, nil
+}
+
+// LoadData 从内存加载IP数据库
+func LoadData(database []byte) {
+	if string(database[6:11]) == "build" {
+		dataType = dataTypeIpdb
+		loadCity, err := ipdb.NewCityFromBytes(database)
+		if err != nil {
+			panic(err)
+		}
+		ipdbCity = loadCity
+		return
+	}
+	data = database
+	dataLen = uint32(len(data))
+}
+
+// LoadFile 从文件加载IP数据库
+func LoadFile(filepath string) (err error) {
+	body, err := os.ReadFile(filepath)
+	if err != nil {
+		return
+	}
+	LoadData(body)
+	return
+}
+
+// SplitResult 按照调整后的纯真社区版IP库地理位置格式返回结果
+func SplitResult(addr string, isp string, ipv4 string) (location *Location) {
+	location = &Location{ISP: isp, IP: ipv4}
+	splitList := strings.Split(addr, "–")
+	for i := 0; i < len(splitList); i++ {
+		switch i {
+		case 0:
+			location.Country = splitList[i]
+		case 1:
+			location.Province = splitList[i]
+		case 2:
+			location.City = splitList[i]
+		case 3:
+			location.District = splitList[i]
+		}
+	}
+	if location.Country == "局域网" {
+		location.ISP = location.Country
+	}
+	return
+}
--- a/common/utils/qqwry/qqwry_test.go
+++ b/common/utils/qqwry/qqwry_test.go
@@ -0,0 +1,32 @@
+package qqwry
+
+import (
+	"testing"
+)
+
+func init() {
+	if err := LoadFile("assets/qqwry.ipdb"); err != nil {
+		panic(err)
+	}
+}
+
+func TestQueryIP(t *testing.T) {
+	queryIp := "119.29.29.29"
+	location, err := QueryIP(queryIp)
+	if err != nil {
+		t.Fatal(err)
+	}
+	emptyVal := func(val string) string {
+		if val != "" {
+			return val
+		}
+		return "未知"
+	}
+	t.Logf("国家：%s，省份：%s，城市：%s，区县：%s，运营商：%s",
+		emptyVal(location.Country),
+		emptyVal(location.Province),
+		emptyVal(location.City),
+		emptyVal(location.District),
+		emptyVal(location.ISP),
+	)
+}
--- a/common/utils/qqwry/server/server.go
+++ b/common/utils/qqwry/server/server.go
@@ -0,0 +1,46 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"github.com/xiaoqidun/qqwry"
+	"github.com/xiaoqidun/qqwry/assets"
+	"net"
+	"net/http"
+)
+
+type resp struct {
+	Data    *qqwry.Location `json:"data"`
+	Success bool            `json:"success"`
+	Message string          `json:"message"`
+}
+
+func init() {
+	qqwry.LoadData(assets.QQWryIpdb)
+}
+
+func main() {
+	listen := flag.String("listen", "127.0.0.1:80", "http server listen addr")
+	flag.Parse()
+	http.HandleFunc("/ip/", IpAPI)
+	if err := http.ListenAndServe(*listen, nil); err != nil {
+		panic(err)
+	}
+}
+
+func IpAPI(writer http.ResponseWriter, request *http.Request) {
+	ip := request.URL.Path[4:]
+	if ip == "" {
+		ip, _, _ = net.SplitHostPort(request.RemoteAddr)
+	}
+	response := &resp{}
+	location, err := qqwry.QueryIP(ip)
+	if err != nil {
+		response.Message = err.Error()
+	} else {
+		response.Data = location
+		response.Success = true
+	}
+	b, _ := json.MarshalIndent(response, "", "  ")
+	_, _ = writer.Write(b)
+}