diff --git a/common/utils/qqwry/LICENSE b/common/utils/qqwry/LICENSE new file mode 100644 index 00000000..ccb78596 --- /dev/null +++ b/common/utils/qqwry/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 xiaoqidun + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/common/utils/qqwry/README.md b/common/utils/qqwry/README.md new file mode 100644 index 00000000..27dd1deb --- /dev/null +++ b/common/utils/qqwry/README.md @@ -0,0 +1,71 @@ +# QQWry [![Go Reference](https://pkg.go.dev/badge/github.com/xiaoqidun/qqwry.svg)](https://pkg.go.dev/github.com/xiaoqidun/qqwry) + +Golang QQWry,高性能纯真IP查询库。 + +# 使用须知 + +1. dat格式仅支持ipv4查询。 +2. ipdb格式支持ipv4和ipv6查询。 + +# 使用说明 + +```go +package main + +import ( + "fmt" + "github.com/xiaoqidun/qqwry" +) + +func main() { + // 从文件加载IP数据库 + if err := qqwry.LoadFile("qqwry.ipdb"); err != nil { + panic(err) + } + // 从内存或缓存查询IP + location, err := qqwry.QueryIP("119.29.29.29") + if err != nil { + fmt.Printf("错误:%v\n", err) + return + } + fmt.Printf("国家:%s,省份:%s,城市:%s,区县:%s,运营商:%s\n", + location.Country, + location.Province, + location.City, + location.District, + location.ISP, + ) +} +``` + +# IP数据库 + +- DAT格式:[https://aite.xyz/share-file/qqwry/qqwry.dat](https://aite.xyz/share-file/qqwry/qqwry.dat) +- IPDB格式:[https://aite.xyz/share-file/qqwry/qqwry.ipdb](https://aite.xyz/share-file/qqwry/qqwry.ipdb) + +# 编译说明 + +1. 下载IP数据库并放置于assets目录中。 +2. client和server需要go1.16的内嵌资源特性。 +3. 作为库使用,请直接引包,并不需要go1.16+才能编译。 + +# 数据更新 + +- 由于qqwry.dat缺乏更新,官方czdb格式又难以获得和分发,建议使用ipdb格式。 +- 这里的ipdb格式指metowolf提供的官方czdb格式转换而来的ipdb格式(纯真格式原版)。 + +# 服务接口 + +1. 自行根据需要调整server下源码。 +2. 可以通过-listen参数指定http服务地址。 +3. json api:curl http://127.0.0.1/ip/119.29.29.29 + +# 特别感谢 + +- 感谢[纯真IP库](https://www.cz88.net/)一直坚持为大家提供免费IP数据库。 +- 感谢[yinheli](https://github.com/yinheli)的[qqwry](https://github.com/yinheli/qqwry)项目,为我提供纯真ip库解析算法参考。 +- 感谢[metowolf](https://github.com/metowolf)的[qqwry.ipdb](https://github.com/metowolf/qqwry.ipdb)项目,提供纯真czdb转ipdb数据库。 + +# 授权说明 + +使用本类库你唯一需要做的就是把LICENSE文件往你用到的项目中拷贝一份。 \ No newline at end of file diff --git a/common/utils/qqwry/client/client.go b/common/utils/qqwry/client/client.go new file mode 100644 index 00000000..aac7ea7e --- /dev/null +++ b/common/utils/qqwry/client/client.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "github.com/xiaoqidun/qqwry" + "github.com/xiaoqidun/qqwry/assets" + "os" +) + +func init() { + qqwry.LoadData(assets.QQWryIpdb) +} + +func main() { + if len(os.Args) < 2 { + return + } + queryIp := os.Args[1] + location, err := qqwry.QueryIP(queryIp) + if err != nil { + fmt.Printf("错误:%v\n", err) + return + } + emptyVal := func(val string) string { + if val != "" { + return val + } + return "未知" + } + fmt.Printf("国家:%s,省份:%s,城市:%s,区县:%s,运营商:%s\n", + emptyVal(location.Country), + emptyVal(location.Province), + emptyVal(location.City), + emptyVal(location.District), + emptyVal(location.ISP), + ) +} diff --git a/common/utils/qqwry/go.mod b/common/utils/qqwry/go.mod new file mode 100644 index 00000000..9201c97d --- /dev/null +++ b/common/utils/qqwry/go.mod @@ -0,0 +1,8 @@ +module github.com/xiaoqidun/qqwry + +go 1.20 + +require ( + github.com/ipipdotnet/ipdb-go v1.3.3 + golang.org/x/text v0.8.0 +) diff --git a/common/utils/qqwry/go.sum b/common/utils/qqwry/go.sum new file mode 100644 index 00000000..19743868 --- /dev/null +++ b/common/utils/qqwry/go.sum @@ -0,0 +1,4 @@ +github.com/ipipdotnet/ipdb-go v1.3.3 h1:GLSAW9ypLUd6EF9QNK2Uhxew9Jzs4XMJ9gOZEFnJm7U= +github.com/ipipdotnet/ipdb-go v1.3.3/go.mod h1:yZ+8puwe3R37a/3qRftXo40nZVQbxYDLqls9o5foexs= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= diff --git a/common/utils/qqwry/qqwry.go b/common/utils/qqwry/qqwry.go new file mode 100644 index 00000000..fff50265 --- /dev/null +++ b/common/utils/qqwry/qqwry.go @@ -0,0 +1,235 @@ +package qqwry + +import ( + "bytes" + "encoding/binary" + "errors" + "github.com/ipipdotnet/ipdb-go" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/transform" + "io" + "net" + "os" + "strings" + "sync" +) + +var ( + data []byte + dataLen uint32 + ipdbCity *ipdb.City + dataType = dataTypeDat + locationCache = &sync.Map{} +) + +const ( + dataTypeDat = 0 + dataTypeIpdb = 1 +) + +const ( + indexLen = 7 + redirectMode1 = 0x01 + redirectMode2 = 0x02 +) + +type Location struct { + Country string // 国家 + Province string // 省份 + City string // 城市 + District string // 区县 + ISP string // 运营商 + IP string // IP地址 +} + +func byte3ToUInt32(data []byte) uint32 { + i := uint32(data[0]) & 0xff + i |= (uint32(data[1]) << 8) & 0xff00 + i |= (uint32(data[2]) << 16) & 0xff0000 + return i +} + +func gb18030Decode(src []byte) string { + in := bytes.NewReader(src) + out := transform.NewReader(in, simplifiedchinese.GB18030.NewDecoder()) + d, _ := io.ReadAll(out) + return string(d) +} + +// QueryIP 从内存或缓存查询IP +func QueryIP(ip string) (location *Location, err error) { + if v, ok := locationCache.Load(ip); ok { + return v.(*Location), nil + } + switch dataType { + case dataTypeDat: + return QueryIPDat(ip) + case dataTypeIpdb: + return QueryIPIpdb(ip) + default: + return nil, errors.New("data type not support") + } +} + +// QueryIPDat 从dat查询IP,仅加载dat格式数据库时使用 +func QueryIPDat(ipv4 string) (location *Location, err error) { + ip := net.ParseIP(ipv4).To4() + if ip == nil { + return nil, errors.New("ip is not ipv4") + } + ip32 := binary.BigEndian.Uint32(ip) + posA := binary.LittleEndian.Uint32(data[:4]) + posZ := binary.LittleEndian.Uint32(data[4:8]) + var offset uint32 = 0 + for { + mid := posA + (((posZ-posA)/indexLen)>>1)*indexLen + buf := data[mid : mid+indexLen] + _ip := binary.LittleEndian.Uint32(buf[:4]) + if posZ-posA == indexLen { + offset = byte3ToUInt32(buf[4:]) + buf = data[mid+indexLen : mid+indexLen+indexLen] + if ip32 < binary.LittleEndian.Uint32(buf[:4]) { + break + } else { + offset = 0 + break + } + } + if _ip > ip32 { + posZ = mid + } else if _ip < ip32 { + posA = mid + } else if _ip == ip32 { + offset = byte3ToUInt32(buf[4:]) + break + } + } + if offset <= 0 { + return nil, errors.New("ip not found") + } + posM := offset + 4 + mode := data[posM] + var ispPos uint32 + var addr, isp string + switch mode { + case redirectMode1: + posC := byte3ToUInt32(data[posM+1 : posM+4]) + mode = data[posC] + posCA := posC + if mode == redirectMode2 { + posCA = byte3ToUInt32(data[posC+1 : posC+4]) + posC += 4 + } + for i := posCA; i < dataLen; i++ { + if data[i] == 0 { + addr = string(data[posCA:i]) + break + } + } + if mode != redirectMode2 { + posC += uint32(len(addr) + 1) + } + ispPos = posC + case redirectMode2: + posCA := byte3ToUInt32(data[posM+1 : posM+4]) + for i := posCA; i < dataLen; i++ { + if data[i] == 0 { + addr = string(data[posCA:i]) + break + } + } + ispPos = offset + 8 + default: + posCA := offset + 4 + for i := posCA; i < dataLen; i++ { + if data[i] == 0 { + addr = string(data[posCA:i]) + break + } + } + ispPos = offset + uint32(5+len(addr)) + } + if addr != "" { + addr = strings.TrimSpace(gb18030Decode([]byte(addr))) + } + ispMode := data[ispPos] + if ispMode == redirectMode1 || ispMode == redirectMode2 { + ispPos = byte3ToUInt32(data[ispPos+1 : ispPos+4]) + } + if ispPos > 0 { + for i := ispPos; i < dataLen; i++ { + if data[i] == 0 { + isp = string(data[ispPos:i]) + if isp != "" { + if strings.Contains(isp, "CZ88.NET") { + isp = "" + } else { + isp = strings.TrimSpace(gb18030Decode([]byte(isp))) + } + } + break + } + } + } + location = SplitResult(addr, isp, ipv4) + locationCache.Store(ipv4, location) + return location, nil +} + +// QueryIPIpdb 从ipdb查询IP,仅加载ipdb格式数据库时使用 +func QueryIPIpdb(ip string) (location *Location, err error) { + ret, err := ipdbCity.Find(ip, "CN") + if err != nil { + return + } + location = SplitResult(ret[0], ret[1], ip) + locationCache.Store(ip, location) + return location, nil +} + +// LoadData 从内存加载IP数据库 +func LoadData(database []byte) { + if string(database[6:11]) == "build" { + dataType = dataTypeIpdb + loadCity, err := ipdb.NewCityFromBytes(database) + if err != nil { + panic(err) + } + ipdbCity = loadCity + return + } + data = database + dataLen = uint32(len(data)) +} + +// LoadFile 从文件加载IP数据库 +func LoadFile(filepath string) (err error) { + body, err := os.ReadFile(filepath) + if err != nil { + return + } + LoadData(body) + return +} + +// SplitResult 按照调整后的纯真社区版IP库地理位置格式返回结果 +func SplitResult(addr string, isp string, ipv4 string) (location *Location) { + location = &Location{ISP: isp, IP: ipv4} + splitList := strings.Split(addr, "–") + for i := 0; i < len(splitList); i++ { + switch i { + case 0: + location.Country = splitList[i] + case 1: + location.Province = splitList[i] + case 2: + location.City = splitList[i] + case 3: + location.District = splitList[i] + } + } + if location.Country == "局域网" { + location.ISP = location.Country + } + return +} diff --git a/common/utils/qqwry/qqwry_test.go b/common/utils/qqwry/qqwry_test.go new file mode 100644 index 00000000..365e042e --- /dev/null +++ b/common/utils/qqwry/qqwry_test.go @@ -0,0 +1,32 @@ +package qqwry + +import ( + "testing" +) + +func init() { + if err := LoadFile("assets/qqwry.ipdb"); err != nil { + panic(err) + } +} + +func TestQueryIP(t *testing.T) { + queryIp := "119.29.29.29" + location, err := QueryIP(queryIp) + if err != nil { + t.Fatal(err) + } + emptyVal := func(val string) string { + if val != "" { + return val + } + return "未知" + } + t.Logf("国家:%s,省份:%s,城市:%s,区县:%s,运营商:%s", + emptyVal(location.Country), + emptyVal(location.Province), + emptyVal(location.City), + emptyVal(location.District), + emptyVal(location.ISP), + ) +} diff --git a/common/utils/qqwry/server/server.go b/common/utils/qqwry/server/server.go new file mode 100644 index 00000000..6be73dff --- /dev/null +++ b/common/utils/qqwry/server/server.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "flag" + "github.com/xiaoqidun/qqwry" + "github.com/xiaoqidun/qqwry/assets" + "net" + "net/http" +) + +type resp struct { + Data *qqwry.Location `json:"data"` + Success bool `json:"success"` + Message string `json:"message"` +} + +func init() { + qqwry.LoadData(assets.QQWryIpdb) +} + +func main() { + listen := flag.String("listen", "127.0.0.1:80", "http server listen addr") + flag.Parse() + http.HandleFunc("/ip/", IpAPI) + if err := http.ListenAndServe(*listen, nil); err != nil { + panic(err) + } +} + +func IpAPI(writer http.ResponseWriter, request *http.Request) { + ip := request.URL.Path[4:] + if ip == "" { + ip, _, _ = net.SplitHostPort(request.RemoteAddr) + } + response := &resp{} + location, err := qqwry.QueryIP(ip) + if err != nil { + response.Message = err.Error() + } else { + response.Data = location + response.Success = true + } + b, _ := json.MarshalIndent(response, "", " ") + _, _ = writer.Write(b) +}