csv 读取加载

zhangqijia
1 parent 1b1ad555
Showing 11 changed files with 400 additions and 5 deletions Show diff stats
.gitignore
Makefile
README.md
actions/accountaction.go -> actions/AccountAction.go
actions/roleaction.go -> actions/RoleAction.go
go.mod
go.sum
protos
tools/csvtostruct.go
tools/main.go
utils/utils.go
@@ -2,5 +2,7 @@
 vendor
 bin
 protos
+csvdata
+*.csv
 *.log
 \ No newline at end of file
@@ -13,8 +13,9 @@ account:
 game:
 	go run cmd/game.go
 build:
-	go build -o bin/account account.go
-	go build -o bin/game game.go
+	go build -o bin/account cmd/account.go
+	go build -o bin/game cmd/game.go
+	go build -o bin/test test/client.go
 cert:
 	openssl req \
@@ -29,9 +29,10 @@ $ go install google.golang.org/grpc/cmd/protoc-gen-go-grpc
 ## Usage
-编译 & 运行
+编译 & 运行 游戏服 & 登录服务
 ```shell
-$ make run
+$ make account
+$ make game
 ```
 测试
 ```shell
@@ -3,6 +3,7 @@ module pro2d
 go 1.17
 require (
+	github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	go.etcd.io/etcd/api/v3 v3.5.2
@@ -8,6 +8,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ=
+github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-Subproject commit cee7537250bcee3ddd04e03d2d142d6d1ac6417e
+Subproject commit f39667f517b4d302674707983210d7ac53460c6b
@@ -0,0 +1,350 @@
+package main
+
+import (
+	"encoding/csv"
+	"fmt"
+	"github.com/axgle/mahonia"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"pro2d/csvdata"
+	"reflect"
+	"strconv"
+	"strings"
+	"unicode"
+)
+
+/**
+ * 将excel中的前四列转化为struct
+ * 第一列字段类型		如 int
+ * 第二列字段名称		如 显示顺序
+ * 第三列字段名		如 id
+ * 第四列s,c,all 	s表示服务端使用 c表示客户端使用 all表示都使用
+ */
+
+var (
+	lineNumber           = 4                                     // 每个工作表至少需要读取的行数
+	structBegin          = "type %s struct {\n"                  // 结构体开始
+	structValue          = "    %s %s	`json:\"%s\" client:\"%s\"`" // 结构体的内容
+	structValueForServer = "    %s %s	`json:\"%s\"`"              // 服务端使用的结构体内容
+	structRemarks        = "	 // %s"                             // 结构体备注
+	structValueEnd       = "\n"                                  // 结构体内容结束
+	structEnd            = "}\n"                                 // 结构体结束
+	header               = "package %s\n\r"                      // 文件头
+
+	initpkg				="package csvdata\n\nimport \"sync\"\n\nvar CsvStruct map[string]interface{}\nvar CsvData *sync.Map\n\nfunc init()  {\n\tCsvStruct = make(map[string]interface{})\n\tCsvData = new(sync.Map)\n\n\t%s\n}\n\nfunc GetCsv(key string) interface{}{\n\tif data, ok := CsvData.Load(key); ok {\n\t\treturn data\n\t}\n\treturn nil\n}\nfunc SetCsv(key, value interface{}) {\n\tCsvData.Store(key, value)\n}"
+	initline 			= "CsvStruct[\"%s\"] = %s{}\n"
+)
+
+type Generate struct {
+	savePath string // 生成文件的保存路径
+	data     string // 生成文件的内容
+	allType  string // 文件当中的数据类型
+}
+
+func NewGenerate(savePath, allType string) *Generate {
+	return &Generate{
+		savePath: savePath,
+		allType:  allType,
+	}
+}
+
+// GBK 转 UTF-8
+func GbkToUtf8(s string) string {
+	decoder := mahonia.NewDecoder("GBK")
+	return decoder.ConvertString(s)
+}
+
+// UTF-8 转 GBK
+func Utf8ToGbk(s string) string {
+	decoder := mahonia.NewDecoder("UTF-8")
+	return decoder.ConvertString(s)
+}
+
+func GetGoTByCustomerT(t string) string {
+	switch t {
+	case "IntSlice":
+		return "[]int"
+	case "StringSlice":
+		return "[]string"
+	case "FloatSlice":
+		return "[]float32"
+	default:
+		return t
+	}
+}
+
+func CustomerTypeToGo(allType string) map[string]bool {
+	s := strings.Split(allType, ",")
+	mt := make(map[string]bool)
+	for _, t := range s {
+		mt[GetGoTByCustomerT(t)] = true
+	}
+	return mt
+}
+
+//csv to struct
+func (g *Generate) CsvToStruct(csvpath, filename string) error {
+	csvfile, err := os.Open(csvpath+filename)
+	if err != nil {
+		return fmt.Errorf("ReadExcel|xlsx.OpenFile is err :%v", err)
+	}
+	defer csvfile.Close()
+
+	r := csv.NewReader(csvfile)
+	i := 0
+	sheetData := make([][]string, 0)
+	for {
+		if i >= lineNumber {
+			break
+		}
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		i++
+		sheetData = append(sheetData, record)
+	}
+	name := strings.Replace(filename, ".csv", "", -1)
+	data, err := g.SplicingData(sheetData, name)
+	if err != nil {
+		return fmt.Errorf("fileName:\"%v\" is err:%v", name, err)
+	}
+
+	if data == "" {
+		return fmt.Errorf("ReadExcel|this.data is nil")
+	}
+	err = g.WriteNewFile(name, data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+//csv to mem
+func (g *Generate)CsvToMem(csvpath, filename string) error {
+	csvfile, err := os.Open(csvpath+filename)
+	if err != nil {
+		return fmt.Errorf("ReadExcel|xlsx.OpenFile is err :%v", err)
+	}
+	defer csvfile.Close()
+
+	r := csv.NewReader(csvfile)
+	i := 0
+	sheetData := make([][]string, 0)
+	name := strings.Replace(filename, ".csv", "", -1)
+	for {
+		record, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if i < lineNumber {
+			sheetData = append(sheetData, record)
+		}else {
+			//4行之后
+			fields := sheetData[1]
+			typs := sheetData[2] //int, string
+			//keys := sheetData[3] //all, s, c
+
+			t := reflect.ValueOf(csvdata.CsvStruct[name]).Type()
+			v := reflect.New(t).Elem()
+			for idx, field := range fields {
+				if field == "" || record[idx] == "" {
+					continue
+				}
+
+				f :=  v.FieldByName(firstRuneToUpper(field))
+				switch typs[idx] {
+				case "int":
+					id, err := strconv.Atoi(record[idx])
+					if err != nil {
+						return err
+					}
+					f.SetInt(int64(id))
+				case "string":
+					f.SetString(record[idx])
+				}
+
+				//fmt.Printf("%d, %s: %v, %v\n", idx,  firstRuneToUpper(field), v.Field(idx), v.Interface())
+				csvdata.SetCsv(name, v.Interface())
+			}
+		}
+		i++
+	}
+	return nil
+}
+
+//拼装struct
+func (g *Generate) SplicingData(data [][]string, structName string) (string, error) {
+	if len(data) != lineNumber {
+		return "", fmt.Errorf("SplicingData|sheetName:%v col's len:%d is err", data, len(data))
+	}
+	marks := data[0]
+	fields := data[1]
+	typs := data[2]
+	keys := data[3]
+	structData := fmt.Sprintf(structBegin, firstRuneToUpper(structName))
+
+	for i, field := range fields {
+		err := g.CheckType(typs[i], structName)
+		if err != nil {
+			return "", err
+		}
+		switch keys[i] {
+		case "key","all":
+			structData += fmt.Sprintf(structValue, firstRuneToUpper(field), GetGoTByCustomerT(typs[i]), strings.ToLower(field), strings.ToLower(field))
+			if field != "" {
+				structData += fmt.Sprintf(structRemarks, GbkToUtf8(marks[i]))
+			}
+			structData += fmt.Sprintf(structValueEnd)
+		case "s":
+			structData += fmt.Sprintf(structValueForServer, firstRuneToUpper(field), GetGoTByCustomerT(typs[i]), strings.ToLower(field))
+			if field != "" {
+				structData += fmt.Sprintf(structRemarks,GbkToUtf8(marks[i]))
+			}
+			structData += fmt.Sprintf(structValueEnd)
+		case "c":
+			continue
+		default:
+			return "", fmt.Errorf("SplicingData|keys[%d]:\"%v\" is not in s,c,all", i, keys[i])
+		}
+	}
+	structData += structEnd
+	return structData, nil
+}
+
+// 拼装好的struct写入新的文件
+func (g *Generate) WriteNewFile(filename, data string) error {
+	str := strings.Split(g.savePath, "/")
+	if len(str) == 0 {
+		return fmt.Errorf("WriteNewFile|len(str) is 0")
+	}
+
+	header = fmt.Sprintf(header, str[len(str)-1])
+	data = header + data
+
+	fw, err := os.OpenFile(g.savePath +"/" +filename+".go", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return fmt.Errorf("WriteNewFile|OpenFile is err:%v", err)
+	}
+	defer fw.Close()
+
+	_, err = fw.Write([]byte(data))
+	if err != nil {
+		return fmt.Errorf("WriteNewFile|Write is err:%v", err)
+	}
+
+	return nil
+}
+
+// 拼装好的初始化写入init.go文件
+func (g *Generate) WriteInitPkg(data string) error {
+	str := strings.Split(g.savePath, "/")
+	if len(str) == 0 {
+		return fmt.Errorf("WriteNewFile|len(str) is 0")
+	}
+
+	fw, err := os.OpenFile(g.savePath +"/init.go", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return fmt.Errorf("WriteNewFile|OpenFile is err:%v", err)
+	}
+	defer fw.Close()
+
+	_, err = fw.Write([]byte(fmt.Sprintf(initpkg, data)))
+	if err != nil {
+		return fmt.Errorf("WriteNewFile|Write is err:%v", err)
+	}
+
+	return nil
+}
+
+// 检测解析出来的字段类型是否符合要求
+func (g *Generate) CheckType(dataType, structName string) error {
+	res := strings.Index(g.allType, dataType)
+	if res == -1 {
+		return fmt.Errorf("CheckType|struct:\"%v\" dataType:\"%v\" is not in provide dataType", structName, dataType)
+	}
+	return nil
+}
+
+// 字符串首字母转换成大写
+func firstRuneToUpper(str string) string {
+	data := []byte(str)
+	for k, v := range data {
+		if k == 0 {
+			first := []byte(strings.ToUpper(string(v)))
+			newData := data[1:]
+			data = append(first, newData...)
+			break
+		}
+	}
+	return string(data[:])
+}
+
+// 判断是否存在汉字或者是否为默认的工作表
+func hasChineseOrDefault(r string) bool {
+	if strings.Index(r, "Sheet") != -1 {
+		return true
+	}
+	for _, v := range []rune(r) {
+		if unicode.Is(unicode.Han, v) {
+			return true
+		}
+	}
+	return false
+}
+
+// 读取Csv 转化为 struct
+func (g *Generate) ReadCsvToStruct(readPath string) error {
+	files, err := ioutil.ReadDir(readPath)
+	if err != nil {
+		return fmt.Errorf("ReadExcel|ReadDir is err:%v", err)
+	}
+
+
+	data := ""
+	for _, file := range files {
+		if path.Ext(file.Name()) != ".csv" || hasChineseOrDefault(file.Name()) {
+			continue
+		}
+		err := g.CsvToStruct(readPath, file.Name())
+		if err != nil {
+			return err
+		}
+
+		name := strings.Replace(file.Name(), ".csv", "", -1)
+		data += fmt.Sprintf(initline, name, firstRuneToUpper(name))
+	}
+	g.WriteInitPkg(data)
+	return nil
+}
+
+// 读取Csv 转化为 memory数据
+func (g *Generate) ReadCsvToMemory(readPath string) error {
+	files, err := ioutil.ReadDir(readPath)
+	if err != nil {
+		return fmt.Errorf("ReadExcel|ReadDir is err:%v", err)
+	}
+
+	for _, file := range files {
+		if path.Ext(file.Name()) != ".csv" || hasChineseOrDefault(file.Name()) {
+			continue
+		}
+		err := g.CsvToMem(readPath, file.Name())
+		if err != nil {
+			return err
+		}
+
+	}
+	return nil
+}
@@ -0,0 +1,29 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	_ "pro2d/csvdata"
+)
+
+var (
+	savePath = flag.String("savePath", "/Users/mac/Documents/project/Pro2D/Pro2DServer/csvdata", "Path to save the makefile")
+	readPath = flag.String("readPath", "/Users/mac/Documents/project/Pro2D/Pro2DServer/tools/", "The path of reading Excel")
+	allType  = flag.String("allType", "int, string", "Specified field type")
+)
+
+func main() {
+	flag.Parse()
+	if *savePath == "" || *readPath == "" || *allType == "" {
+		fmt.Println("savePath, readPath or allType is nil")
+		return
+	}
+
+	gt := NewGenerate(*savePath, *allType)
+	//err := gt.ReadCsvToStruct(*readPath)
+	err := gt.ReadCsvToMemory(*readPath)
+	if err != nil {
+		fmt.Printf("something err:%v\n", err)
+		return
+	}
+}
 \ No newline at end of file
@@ -44,4 +44,13 @@ func FindIndex(schema interface{}) (string, []string){
 		}
 	}
 	return strings.ToLower(s.Name()), index
+}
+
+func GetIdxBySlice(s []interface{}, i interface{}) (int){
+	for idx, v := range s {
+		if v ==  i {
+			return idx
+		}
+	}
+	return -1
 }
 \ No newline at end of file