add phonemizers (transformer) and their dataset

Author: neurlang

cmd/train_phonemizer/main.go

@@ -11,9 +11,10 @@ import "time"
 
 import "github.com/neurlang/classifier/datasets/phonemizer"
 //import "github.com/neurlang/classifier/layer/majpool2d"
-//import "github.com/neurlang/classifier/layer/sum"
-//import "github.com/neurlang/classifier/layer/sochastic"
-import "github.com/neurlang/classifier/layer/parity"
+import "github.com/neurlang/classifier/layer/sum"
+import "github.com/neurlang/classifier/layer/sochastic"
+//import "github.com/neurlang/classifier/layer/parity"
+import "github.com/neurlang/classifier/layer/crossattention"
 import "github.com/neurlang/classifier/datasets"
 import "github.com/neurlang/classifier/hashtron"
 //import "github.com/neurlang/classifier/learning"
@@ -31,9 +32,13 @@ func error_abs(a, b uint32) uint32 {
 func main() {
 cleantsv := flag.String("cleantsv", "", "clean tsv dataset for the language")
 premodulo := flag.Int("premodulo", 0, "premodulo")
+minpremodulo := flag.Int("minpremodulo", 0, "minpremodulo")
+maxpremodulo := flag.Int("maxpremodulo", 0, "maxpremodulo")
+maxdepth := flag.Int("maxdepth", 0, "max training depth")
 part := flag.Int("part", 0, "train on n/part-th")
 dstmodel := flag.String("dstmodel", "", "model destination .json.lzw file")
 flag.Bool("pgo", false, "enable pgo")
+boosting := flag.Bool("padspace", false, "enable padspace")
 resume := flag.Bool("resume", false, "resume training")
 flag.Parse()
 
@@ -43,58 +48,35 @@ func main() {
 println("clean tsv is mandatory")
 return
 }
+if maxdepth == nil || *maxdepth == 0 {
+println("max depth is mandatory")
+return
+}
 
-data := phonemizer.Split(phonemizer.NewDataset(*cleantsv))
+data := phonemizer.Split(phonemizer.NewDataset(*cleantsv, boosting != nil && *boosting))
 
 if len(data) == 0 {
 println("it looks like no data for this language, or language is unambiguous (no model needed)")
 return
 }
-/*
-
-
-
-net.NewLayer(fanout1*fanout2*fanout3, 0)
-net.NewCombiner(sochastic.MustNew(fanout1*fanout2*fanout3, 32, 1))
-net.NewLayer(fanout1*fanout2, 0)
 
-*/
-const fanout1 = 5
-var net feedforward.FeedforwardNetwork
-//net.NewLayer(fanout1, 0)
-//net.NewCombiner(sochastic.MustNew(fanout1, 32, 0))
-net.NewLayer(fanout1, 0)
-net.NewCombiner(parity.MustNew(fanout1))
-net.NewLayer(1, 0)
-/*
-net.NewCombiner(sochastic.MustNew(1, 32, 0))
-net.NewLayerPI(1, 0, 0)
-net.NewCombiner(sochastic.MustNew(1, 32, 0))
-net.NewLayerPI(1, 0, 0)
-net.NewCombiner(sochastic.MustNew(1, 1, 0))
-*/
-/*
-const fanout1 = 1
-const fanout2 = 5
+const fanout1 = 32
+const fanout2 = 4
 const fanout3 = 3
-const fanout4 = 5
-//const fanout5 = 1
-//const fanout6 = 4
-//const fanout7 = 1
-//const fanout8 = 5
-
+
 var net feedforward.FeedforwardNetwork
-//net.NewLayerP(fanout1*fanout2*fanout3*fanout4*fanout5*fanout6*fanout7*fanout8, 0, 1<<fanout8)
-//net.NewCombiner(majpool2d.MustNew2(fanout1*fanout2*fanout3*fanout4*fanout5*fanout6*fanout8, 1, fanout7, 1, fanout8, 1, 1, 0))
-//net.NewLayerP(fanout1*fanout2*fanout3*fanout4*fanout5*fanout6, 0, 1<<(fanout6*fanout6*2/3))
-//net.NewCombiner(majpool2d.MustNew2(fanout1*fanout2*fanout3*fanout4*fanout6, 1, fanout5, 1, fanout6, 1, 1, 0))
-net.NewLayerP(fanout1*fanout2*fanout3*fanout4, 0, 1<<13)
-net.NewCombiner(majpool2d.MustNew2(fanout1*fanout2*fanout4, 1, fanout3, 1, fanout4, 1, 1, 0))
 net.NewLayer(fanout1*fanout2, 0)
-//net.NewCombiner(full.MustNew(fanout2, 1, 1))
-net.NewCombiner(majpool2d.MustNew2(fanout2, 1, fanout1, 1, fanout2, 1, 1, 0))
+for i := 0; i < fanout3; i++ {
+net.NewCombiner(crossattention.MustNew(fanout1, fanout2))
+net.NewLayerPI(fanout1*fanout2, 0, 0)
+net.NewCombiner(sochastic.MustNew(fanout1*fanout2, 8*byte(i), uint32(i)))
+net.NewLayerPI(fanout1*fanout2, 0, 0)
+}
+net.NewCombiner(sochastic.MustNew(fanout1*fanout2, 32, fanout3))
+net.NewLayer(fanout1*fanout2, 0)
+net.NewCombiner(sum.MustNew([]uint{fanout1*fanout2}, 0))
 net.NewLayer(1, 0)
-*/
+
 
 trainWorst := func(worst int) func() {
 var tally = new(datasets.Tally)
@@ -103,7 +85,16 @@ func main() {
 if premodulo != nil && *premodulo > 0 {
 tally.SetGlobalPremodulo(uint32(*premodulo))
 }
-
+if minpremodulo != nil && *minpremodulo > 0 && maxpremodulo != nil && *maxpremodulo > 0 {
+const span = 50 * 50
+value := (100 - improved_success_rate) * (100 - improved_success_rate)
+premodulo := value * ( *minpremodulo - *maxpremodulo ) / span + *maxpremodulo
+//println(improved_success_rate, premodulo)
+if premodulo < 2 {
+premodulo = 2
+}
+tally.SetGlobalPremodulo(uint32(premodulo))
+}
 var parts = 1
 if part != nil && *part > 1 {
 rand.Seed(time.Now().UnixNano())
@@ -113,7 +104,7 @@ func main() {
 
 parallel.ForEach(len(data)/parts, 1000, func(jjj int) {
 {
-var io = data[jjj].V1()
+var io = data[jjj].V2(fanout1)
 
 net.Tally4(io, worst, tally, nil)
 }
@@ -122,41 +113,13 @@ func main() {
 if !tally.GetImprovementPossible() {
 return nil
 }
-/*
-var h learning.HyperParameters
-h.Threads = runtime.NumCPU()
-h.Factor = 1 // affects the solution size
-
-// shuffle before solving attempts
-h.Shuffle = true
-h.Seed = true
-
-// restart when stuck
-h.DeadlineMs = 1000
-h.DeadlineRetry = 10
 
-// affects how fast is the modulo reduced
-h.Subtractor = 1
-
-// reduce Backtracking printing on the log
-h.Printer = 70
-
-// save any solution to disk
-h.InitialLimit = 1000 + 4*tally.Len()
-h.EndWhenSolved = true
-
-h.Name = fmt.Sprint(worst)
-//h.SetLogger("solutions11.txt")
-
-//h.AvxLanes = 16
-//h.AvxSkip = 4
-*/
 fmt.Println("hashtron position:", worst, "(job size:", tally.Len(), ")")
 ptr := net.GetHashtron(worst)
 dset := tally.Dataset()
 q := quaternary.Make(dset)
 var pmod = [][2]uint32{}
-if premodulo != nil && *premodulo > 0 {
+if (premodulo != nil && *premodulo > 0) || (minpremodulo != nil && *minpremodulo > 0 && maxpremodulo != nil && *maxpremodulo > 0) {
 pmod = [][2]uint32{tally.GetGlobalSaltPremodulo()}
 }
 htron, err := hashtron.New(pmod, ptr.Bits(), []byte(q))
@@ -182,7 +145,7 @@ func main() {
 var percent, errsum atomic.Uint64
 parallel.ForEach(len(data)/parts, 1000, func(j int) {
 {
-var io = data[j].V1()
+var io = data[j].V2(fanout1)
 
 var predicted = net.Infer2(io) & 1
 
@@ -206,7 +169,7 @@ func main() {
 
 if dstmodel != nil && len(*dstmodel) > 0 && improved_success_rate < success {
 if improved_success_rate > 0 {
-model := strings.ReplaceAll(*dstmodel, "weights1", "weights2")
+model := strings.ReplaceAll(*dstmodel, "weights1", "weights4")
 err := net.WriteZlibWeightsToFile(model)
 if err != nil {
 println(err.Error())
@@ -222,7 +185,7 @@ func main() {
 return success, h.Sum()
 }
 if resume != nil && *resume && dstmodel != nil {
-model := strings.ReplaceAll(*dstmodel, "weights1", "weights2")
+model := strings.ReplaceAll(*dstmodel, "weights1", "weights4")
 
 err := net.ReadZlibWeightsFromFile(model)
 if err != nil {
@@ -244,7 +207,7 @@ func main() {
 if m.Exists(state, shuf[0], byte(success)) {
 continue
 }
-for worst := 0; worst < len(shuf); worst++ {
+for worst := 0; worst < len(shuf) && worst < *maxdepth; worst++ {
 println("training #", worst, "hastron of", len(shuf), "hashtrons total")
 if this_backoff := trainWorst(shuf[worst]); this_backoff != nil {
 infloop = -1