Add datasets anytally, pretally, add trainer

Author: neurlang

datasets/anytally.go

@@ -0,0 +1,49 @@
+package datasets
+
+type AnyTally interface {
+
+// Erase
+Init()
+
+// Global Premodulo
+IsGlobalPremodulo() bool
+SetGlobalPremodulo(mod uint32)
+GetGlobalSaltPremodulo() [2]uint32
+GetGlobalPremodulo() uint32
+
+// Pre tallying
+GetCellDecision(position int, feature uint32) (bool, bool)
+SetCellDecision(position int, feature uint32, output bool)
+
+// Tallying
+AddToCorrect(feature uint32, vote int8, improvement bool)
+AddToImprove(feature uint32, vote int8)
+AddToMapping(feature uint16, output uint64)
+
+// Get Dataset at
+DatasetAt(n int) Dataset
+GetImprovementPossible() bool
+
+// Len
+Len() (ret int)
+}
+
+type TallyType byte
+const PreTallyType TallyType = 2
+const FinTallyType TallyType = 1
+
+func NewAnyTally(typ TallyType) AnyTally {
+switch typ {
+case PreTallyType:
+t := &PreTally{}
+t.Init()
+return t
+case FinTallyType:
+t := &Tally{}
+t.Init()
+t.SetFinalization(true)
+return t
+default:
+return nil
+}
+}

datasets/pretally.go

@@ -0,0 +1,137 @@
+package datasets
+
+import "sync"
+import "crypto/rand"
+import "encoding/binary"
+
+// PreTally stores distilled decisions for multiple cells with thread-safe access
+type PreTally struct {
+maps []map[uint32]bool
+mutex sync.RWMutex
+
+// global premodulo and salt
+globalPremodulo, globalSalt uint32
+
+}
+
+
+// Init resets pretally to be empty
+func (d *PreTally) Init() {
+d.mutex.Lock()
+defer d.mutex.Unlock()
+d.maps = nil
+}
+
+
+func (t *PreTally) IsGlobalPremodulo() bool {
+return t.globalPremodulo != 0
+}
+func (t *PreTally) SetGlobalPremodulo(mod uint32) {
+var b [4]byte
+rand.Read(b[:])
+t.globalSalt = binary.LittleEndian.Uint32(b[:])
+t.globalPremodulo = mod
+}
+func (t *PreTally) GetGlobalSaltPremodulo() [2]uint32 {
+return [2]uint32{t.globalSalt, t.globalPremodulo}
+}
+func (t *PreTally) GetGlobalPremodulo() uint32 {
+return t.globalPremodulo
+}
+
+// GetCellDecision returns the distilled output for a specific cell and feature
+func (d *PreTally) GetCellDecision(position int, feature uint32) (bool, bool) {
+if position < 0 {
+return false, false
+}
+
+d.mutex.RLock()
+defer d.mutex.RUnlock()
+
+if position >= len(d.maps) {
+return false, false
+}
+
+
+val, exists := d.maps[position][feature]
+return val, exists
+}
+
+// SetCellDecision stores a distilled decision for a specific cell and feature
+func (d *PreTally) SetCellDecision(position int, feature uint32, output bool) {
+if position < 0 {
+return
+}
+
+d.mutex.Lock()
+defer d.mutex.Unlock()
+
+for position >= len(d.maps) {
+d.maps = append(d.maps, make(map[uint32]bool))
+}
+
+d.maps[position][feature] = output
+}
+
+func (d *PreTally) Len() (ret int) {
+d.mutex.RLock()
+defer d.mutex.RUnlock()
+
+for _, m := range d.maps {
+ret += len(m)
+}
+return
+}
+func (d *PreTally) Free() {
+d.mutex.Lock()
+defer d.mutex.Unlock()
+
+d.maps = nil
+}
+func (d *PreTally) DatasetAt(position int) Dataset {
+if position < 0 {
+return nil
+}
+d.mutex.RLock()
+defer d.mutex.RUnlock()
+if position >= len(d.maps) {
+return nil
+}
+return d.maps[position]
+}
+
+// GetImprovementPossible reads improvementPossible
+func (t *PreTally) GetImprovementPossible() bool {
+t.mutex.RLock()
+defer t.mutex.RUnlock()
+for _, m := range t.maps {
+if len(m) > 0 {
+return true
+}
+}
+return false
+}
+// AddToCorrect votes for feature which caused the overall result to be correct
+func (t *PreTally) AddToCorrect(feature uint32, vote int8, improvement bool) {
+t.mutex.Lock()
+defer t.mutex.Unlock()
+
+if len(t.maps) == 0 {
+t.maps = append(t.maps, make(map[uint32]bool))
+}
+t.maps[0][feature] = vote > 0
+}
+// AddToImprove votes for feature which caused the overall result to be correct
+func (t *PreTally) AddToImprove(feature uint32, vote int8) {
+t.mutex.Lock()
+defer t.mutex.Unlock()
+
+if len(t.maps) == 0 {
+t.maps = append(t.maps, make(map[uint32]bool))
+}
+t.maps[0][feature] = vote > 0
+}
+// AddToMapping adds feature maps to this output votes to mapping
+func (t *PreTally) AddToMapping(feature uint16, output uint64) {
+// not supported
+}

trainer/evaluatefunc.go

@@ -0,0 +1,109 @@
+package trainer
+
+import "fmt"
+import "os"
+import "math"
+
+import "github.com/neurlang/classifier/parallel"
+import "github.com/neurlang/classifier/net/feedforward"
+
+type dummy struct{}
+
+func (d dummy) MustPutUint16(n int, value uint16) {}
+func (d dummy) Sum() [32]byte {
+return [32]byte{}
+}
+
+type EvaluateFuncHasher interface {
+MustPutUint16(n int, value uint16)
+Sum() [32]byte
+}
+
+// sampleSize calculates the statistically sufficient sample size
+// for a given dataset size N and significance level (0–100).
+func sampleSize(N int, significance byte) int {
+
+// Convert significance level to Z-score
+z := zScoreFromAlpha(100 - significance)
+
+// Assume worst-case proportion p = 0.5 for max variability
+p := 0.5
+e := float64(100 - significance) // Margin of error = 5%
+
+numerator := math.Pow(z, 2) * p * (1 - p)
+denominator := math.Pow(e, 2)
+
+// Initial sample size without population correction
+ss := numerator / denominator
+
+// Apply finite population correction
+correctedSS := ss * float64(N) / (float64(N) - 1 + ss)
+
+if int(correctedSS) > N {
+return N
+}
+
+return int(correctedSS)
+}
+
+// zScoreFromAlpha returns the Z-score for a given alpha level
+// Common: 90% => 1.645, 95% => 1.96, 99% => 2.576
+func zScoreFromAlpha(alpha byte) float64 {
+switch {
+case alpha <= 1:
+return 2.576 // 99% confidence
+case alpha <= 5:
+return 1.96 // 95% confidence
+case alpha <= 10:
+return 1.645 // 90% confidence
+default:
+return 1.96 // default fallback
+}
+}
+
+func NewEvaluateFunc(net feedforward.FeedforwardNetwork, length int, significance byte, succ *int, dstmodel *string,
+testFunc func(portion int, h EvaluateFuncHasher) int) func() (int, [32]byte) {
+
+return func() (int, [32]byte) {
+var h dummy
+var ha EvaluateFuncHasher = h
+var success int
+if length != 0 {
+length = sampleSize(length, significance)
+hsh := parallel.NewUint16Hasher(length)
+ha = hsh
+success = testFunc(length, hsh)
+} else {
+success = testFunc(0, h)
+}
+
+if dstmodel == nil || *dstmodel == "" {
+err := net.WriteZlibWeightsToFile("output." + fmt.Sprint(success) + ".json.t.lzw")
+if err != nil {
+println(err.Error())
+}
+}
+
+if dstmodel != nil && len(*dstmodel) > 0 && ((succ != nil && (*succ < success || success == 99)) || succ == nil) {
+if succ != nil && *succ > 0 {
+err := net.WriteZlibWeightsToFile(*dstmodel)
+if err != nil {
+println(err.Error())
+}
+}
+if succ != nil {
+*succ = success
+}
+} else if dstmodel != nil && len(*dstmodel) > 0 {
+if succ != nil {
+*succ = success
+}
+}
+
+if success >= 100 {
+println("Max accuracy or wrong data. Exiting")
+os.Exit(0)
+}
+return success, ha.Sum()
+}
+}

trainer/loopfunc.go

@@ -0,0 +1,75 @@
+package trainer
+
+import "os"
+import "fmt"
+import "math/rand"
+import "time"
+
+import "github.com/neurlang/classifier/net/feedforward"
+import "github.com/neurlang/classifier/parallel"
+
+func NewLoopFunc(net feedforward.FeedforwardNetwork, succ *int, treshold int, evaluate func() (int, [32]byte), trainWorst func([]int, int) (undo func())) func() {
+
+var m = parallel.NewMoveSet()
+var success, state = evaluate()
+var default_backoff = func() {
+println("Infinite loop - algorithm stuck in local minimum. Exiting")
+os.Exit(0)
+}
+backoff := default_backoff
+var local_minimums = make(map[[32]byte]struct{})
+fmt.Printf("%x\n", state)
+for {
+for infloop := 0; infloop < net.Len(); infloop++ {
+var shuf []int
+if success < treshold {
+shuf = net.Sequence(false)
+rand.Seed(time.Now().UnixNano())
+rand.Shuffle(len(shuf), func(i, j int) { shuf[i], shuf[j] = shuf[j], shuf[i] })
+} else {
+shuf = net.Branch(false)
+}
+if m.Exists(state, shuf[0], byte(success)) {
+continue
+}
+for worst := 0; worst < len(shuf); worst++ {
+println("training #", worst, "hastron of", len(shuf), "hashtrons total")
+inSucc := success
+if succ != nil {
+inSucc = *succ
+}
+worsts := []int{shuf[worst]}
+if inSucc < treshold {
+if worst+1 < len(shuf) {
+worsts = append(worsts, shuf[worst+1])
+} else {
+break
+}
+}
+if this_backoff := trainWorst(worsts, inSucc); this_backoff != nil {
+infloop = -1
+this_success, this_state := evaluate()
+if _, bad := local_minimums[this_state]; bad {
+this_backoff()
+break
+} else {
+backoff, success, state = this_backoff, this_success, this_state
+}
+} else if worst == 0 {
+break
+}
+fmt.Printf("%x\n", state)
+m.Insert(state, shuf[worst], byte(success))
+if worst != len(shuf)-1 {
+if m.Exists(state, shuf[worst+1], byte(success)) {
+break
+}
+}
+}
+}
+local_minimums[state] = struct{}{}
+backoff()
+backoff = default_backoff
+success, state = evaluate()
+}
+}

trainer/resume.go

@@ -0,0 +1,12 @@
+package trainer
+
+import "github.com/neurlang/classifier/net/feedforward"
+
+func Resume(net feedforward.FeedforwardNetwork, resume *bool, dstmodel *string) {
+if resume != nil && *resume && dstmodel != nil {
+err := net.ReadZlibWeightsFromFile(*dstmodel)
+if err != nil {
+println(err.Error())
+}
+}
+}