Add all keras optimizers

Add custom class weighting Add configurable losses and binary crossentropy loss Remove batch size from model compile method Move metrics to >= for positive Add binary metrics Fix training=True for evaluation bug

Author: CodingBeard

data/single_file_dataset.go

@@ -74,6 +74,7 @@ type SingleFileDatasetConfig struct {
 RowFilter func(line []string) bool
 ConcurrentFileLimit int32
 MaxRowsForProcessorFit int
+ClassWeights map[int]float32
 }
 
 func NewSingleFileDataset(
@@ -97,6 +98,10 @@ func NewSingleFileDataset(
 config.MaxRowsForProcessorFit = 1000000
 }
 
+if config.ClassWeights == nil {
+config.ClassWeights = make(map[int]float32)
+}
+
 var openFileCount int32
 var generatorOffset int32
 
@@ -114,7 +119,7 @@ func NewSingleFileDataset(
 valPercent: config.ValPercent,
 testPercent: config.TestPercent,
 ClassCounts: make(map[int]int),
-ClassWeights: make(map[int]float32),
+ClassWeights: config.ClassWeights,
 filter: config.RowFilter,
 concurrentFileLimit: config.ConcurrentFileLimit,
 openFileCount: &openFileCount,
@@ -201,7 +206,9 @@ func (d *SingleFileDataset) readLineOffsets() error {
 d.lineOffsets = cache.LineOffsets
 d.Count = cache.Count
 d.ClassCounts = cache.ClassCounts
-d.ClassWeights = cache.ClassWeights
+if len(d.ClassWeights) == 0 {
+d.ClassWeights = cache.ClassWeights
+}
 
 d.logger.InfoF("data", "Found %d rows. Got class counts: %#v Got class weights: %#v", d.Count, d.ClassCounts, d.ClassWeights)
 
@@ -308,14 +315,16 @@ func (d *SingleFileDataset) readLineOffsets() error {
 swg.Wait()
 fmt.Println()
 
-majorClassCount := 0
-for _, count := range d.ClassCounts {
-if count > majorClassCount {
-majorClassCount = count
+if len(d.ClassWeights) == 0 {
+majorClassCount := 0
+for _, count := range d.ClassCounts {
+if count > majorClassCount {
+majorClassCount = count
+}
+}
+for class, count := range d.ClassCounts {
+d.ClassWeights[class] = float32(majorClassCount) / float32(count)
 }
-}
-for class, count := range d.ClassCounts {
-d.ClassWeights[class] = float32(majorClassCount) / float32(count)
 }
 
 cacheBytes, e := json.Marshal(fileStatsCache{

examples/class_weights/main.go

@@ -10,6 +10,7 @@ import (
 "github.com/codingbeard/tfkg/layer"
 "github.com/codingbeard/tfkg/metric"
 "github.com/codingbeard/tfkg/model"
+"github.com/codingbeard/tfkg/optimizer"
 "github.com/codingbeard/tfkg/preprocessor"
 tf "github.com/galeone/tensorflow/tensorflow/go"
 "math/rand"
@@ -107,9 +108,7 @@ func main() {
 
 // This part is pretty nasty under the hood. Effectively it will generate some python code for our model and execute it to save the model in a format we can load and train
 // A python binary must be available to use for this to work
-// The batchSize MUST match the batchSize in the call to Fit or Evaluate
-batchSize := 1000
-e = m.CompileAndLoad(batchSize, logsDir)
+e = m.CompileAndLoad(model.LossSparseCategoricalCrossentropy, optimizer.NewAdam(), logsDir)
 if e != nil {
 return
 }
@@ -130,7 +129,7 @@ func main() {
 model.FitConfig{
 Epochs: 10,
 Validation: true,
-BatchSize: batchSize,
+BatchSize: 1000,
 PreFetch: 10,
 Verbose: 1,
 Metrics: []metric.Metric{

examples/iris/main.go

@@ -10,6 +10,7 @@ import (
 "github.com/codingbeard/tfkg/layer"
 "github.com/codingbeard/tfkg/metric"
 "github.com/codingbeard/tfkg/model"
+"github.com/codingbeard/tfkg/optimizer"
 "github.com/codingbeard/tfkg/preprocessor"
 tf "github.com/galeone/tensorflow/tensorflow/go"
 "os"
@@ -109,9 +110,7 @@ func main() {
 
 // This part is pretty nasty under the hood. Effectively it will generate some python code for our model and execute it to save the model in a format we can load and train
 // A python binary must be available to use for this to work
-// The batchSize MUST match the batchSize in the call to Fit or Evaluate
-batchSize := 3
-e = m.CompileAndLoad(batchSize, saveDir)
+e = m.CompileAndLoad(model.LossSparseCategoricalCrossentropy, optimizer.NewAdam(), saveDir)
 if e != nil {
 return
 }
@@ -133,7 +132,7 @@ func main() {
 model.FitConfig{
 Epochs: 10,
 Validation: true,
-BatchSize: batchSize,
+BatchSize: 3,
 PreFetch: 10,
 Verbose: 1,
 Metrics: []metric.Metric{

examples/jobs/main.go

@@ -10,6 +10,7 @@ import (
 "github.com/codingbeard/tfkg/layer"
 "github.com/codingbeard/tfkg/metric"
 "github.com/codingbeard/tfkg/model"
+"github.com/codingbeard/tfkg/optimizer"
 "github.com/codingbeard/tfkg/preprocessor"
 tf "github.com/galeone/tensorflow/tensorflow/go"
 "os"
@@ -271,12 +272,11 @@ func main() {
 layer.DenseWithActivation("swish"),
 )(mergedDense1)
 
-// Get the number of classes from the dataset if we don't want to count them manually, but in this case it is only 2
 output := layer.NewDense(
-float64(dataset.NumCategoricalClasses()),
+1,
 layer.DenseWithDtype(layer.Float32),
 layer.DenseWithName("output"),
-layer.DenseWithActivation("softmax"),
+layer.DenseWithActivation("sigmoid"),
 )(mergedDense2)
 
 // Define a keras style Functional model
@@ -289,9 +289,7 @@ func main() {
 
 // This part is pretty nasty under the hood. Effectively it will generate some python code for our model and execute it to save the model in a format we can load and train
 // A python binary must be available to use for this to work
-// The batchSize MUST match the batch size in the call to Fit or Evaluate
-batchSize := 200
-e = m.CompileAndLoad(batchSize, saveDir)
+e = m.CompileAndLoad(model.LossBinaryCrossentropy, optimizer.NewAdam(), saveDir)
 if e != nil {
 return
 }
@@ -313,7 +311,7 @@ func main() {
 model.FitConfig{
 Epochs: 10,
 Validation: true,
-BatchSize: batchSize,
+BatchSize: 200,
 PreFetch: 10,
 Verbose: 1,
 Metrics: []metric.Metric{

examples/multiple_inputs/main.go

@@ -10,6 +10,7 @@ import (
 "github.com/codingbeard/tfkg/layer"
 "github.com/codingbeard/tfkg/metric"
 "github.com/codingbeard/tfkg/model"
+"github.com/codingbeard/tfkg/optimizer"
 "github.com/codingbeard/tfkg/preprocessor"
 tf "github.com/galeone/tensorflow/tensorflow/go"
 "os"
@@ -111,7 +112,7 @@ func main() {
 // This part is pretty nasty under the hood. Effectively it will generate some python code for our model and execute it to save the model in a format we can load and train
 // A python binary must be available to use for this to work
 // The batchSize MUST match the batch size in the call to Fit or Evaluate
-e = m.CompileAndLoad(3, saveDir)
+e = m.CompileAndLoad(model.LossSparseCategoricalCrossentropy, optimizer.NewAdam(), saveDir)
 if e != nil {
 return
 }

generate/python/generate.go

@@ -36,7 +36,9 @@ func main() {
 }
 for _, object := range objects {
 fmt.Println(object.Type, object.Name)
-if object.Type == "initializer" {
+if object.Type == "optimizer" {
+createOptimizer(object)
+} else if object.Type == "initializer" {
 createInitializer(object)
 } else if object.Type == "regularizer" {
 createRegularizer(object)
@@ -51,6 +53,112 @@ func main() {
 if e != nil {
 panic(e)
 }
+
+_, e = exec.Command("go", "fmt", "github.com/codingbeard/tfkg/layer/constraint").Output()
+if e != nil {
+panic(e)
+}
+
+_, e = exec.Command("go", "fmt", "github.com/codingbeard/tfkg/layer/initializer").Output()
+if e != nil {
+panic(e)
+}
+
+_, e = exec.Command("go", "fmt", "github.com/codingbeard/tfkg/layer/regularizer").Output()
+if e != nil {
+panic(e)
+}
+
+_, e = exec.Command("go", "fmt", "github.com/codingbeard/tfkg/optimizer").Output()
+if e != nil {
+panic(e)
+}
+}
+
+func createOptimizer(object objectJson) {
+e := os.MkdirAll("../../optimizer", os.ModePerm)
+if e != nil {
+panic(e)
+}
+
+var setters []string
+var objectProperties []string
+objectPropertyNames := make(map[string]bool)
+for _, param := range getRequiredParams(object) {
+objectPropertyNames[param[0]] = true
+objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
+}
+for _, param := range getOptionalParams(object) {
+objectPropertyNames[param[0]] = true
+objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
+setters = append(setters, getOptionString(object.Name, param[0], param[1]))
+}
+
+subConfig := object.Config["config"].(map[string]interface{})
+for originalName, value := range subConfig {
+name := snakeCaseToCamelCase(originalName)
+if _, ok := objectPropertyNames[name]; ok {
+continue
+}
+objectProperties = append(objectProperties, fmt.Sprintf("%s %s", snakeCaseToCamelCase(name), getGolangTypeFromValue(object.Name, originalName, value)))
+}
+
+var requiredParamSetters []string
+for _, paramName := range getRequiredParamNames(object) {
+requiredParamSetters = append(requiredParamSetters, fmt.Sprintf("%s: %s", paramName, paramName))
+}
+
+var defaultParamSetters []string
+for _, param := range getOptionalParamDefaults(object) {
+defaultParamSetters = append(defaultParamSetters, fmt.Sprintf("%s: %s,", param[0], param[1]))
+}
+
+lines := []string{
+"package optimizer",
+"",
+fmt.Sprintf("type %s struct {", object.Name),
+"\t" + strings.Join(objectProperties, "\n\t"),
+"}",
+"",
+fmt.Sprintf("func New%s(%s) *%s {", object.Name, getRequiredParamsString(object), object.Name),
+fmt.Sprintf(
+"\treturn &%s{\n\t\t%s%s\t\n\t}",
+object.Name,
+strings.Join(requiredParamSetters, "\n\t\t"),
+strings.Join(defaultParamSetters, "\n\t\t"),
+),
+"}",
+"",
+strings.Join(setters, "\n\n"),
+"",
+fmt.Sprintf("type jsonConfig%s struct {", object.Name),
+"\tClassName string `json:\"class_name\"`",
+"\tName string `json:\"name\"`",
+"\tConfig map[string]interface{} `json:\"config\"`",
+"}",
+fmt.Sprintf(
+"func (%s *%s) GetKerasLayerConfig() interface{} {",
+strings.ToLower(string(object.Name[0])),
+object.Name,
+),
+fmt.Sprintf("\tif %s == nil {", strings.ToLower(string(object.Name[0]))),
+"\t\treturn nil",
+"\t}",
+fmt.Sprintf("\treturn jsonConfig%s{", object.Name),
+fmt.Sprintf("\t\tClassName: \"%s\",", object.Config["class_name"]),
+fmt.Sprintf("\t\tConfig: %s,", getConfigValue(object)),
+"\t}",
+"}",
+}
+
+e = ioutil.WriteFile(
+filepath.Join("../../optimizer", fmt.Sprintf("%s.go", object.Name)),
+[]byte(strings.Join(lines, "\n")),
+os.ModePerm,
+)
+if e != nil {
+panic(e)
+}
 }
 
 func createInitializer(object objectJson) {
@@ -61,14 +169,26 @@ func createInitializer(object objectJson) {
 
 var setters []string
 var objectProperties []string
+objectPropertyNames := make(map[string]bool)
 for _, param := range getRequiredParams(object) {
+objectPropertyNames[param[0]] = true
 objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
 }
 for _, param := range getOptionalParams(object) {
+objectPropertyNames[param[0]] = true
 objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
 setters = append(setters, getOptionString(object.Name, param[0], param[1]))
 }
 
+subConfig := object.Config["config"].(map[string]interface{})
+for originalName, value := range subConfig {
+name := snakeCaseToCamelCase(originalName)
+if _, ok := objectPropertyNames[name]; ok {
+continue
+}
+objectProperties = append(objectProperties, fmt.Sprintf("%s %s", snakeCaseToCamelCase(name), getGolangTypeFromValue(object.Name, originalName, value)))
+}
+
 var requiredParamSetters []string
 for _, paramName := range getRequiredParamNames(object) {
 requiredParamSetters = append(requiredParamSetters, fmt.Sprintf("%s: %s", paramName, paramName))
@@ -135,14 +255,26 @@ func createRegularizer(object objectJson) {
 
 var setters []string
 var objectProperties []string
+objectPropertyNames := make(map[string]bool)
 for _, param := range getRequiredParams(object) {
+objectPropertyNames[param[0]] = true
 objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
 }
 for _, param := range getOptionalParams(object) {
+objectPropertyNames[param[0]] = true
 objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
 setters = append(setters, getOptionString(object.Name, param[0], param[1]))
 }
 
+subConfig := object.Config["config"].(map[string]interface{})
+for originalName, value := range subConfig {
+name := snakeCaseToCamelCase(originalName)
+if _, ok := objectPropertyNames[name]; ok {
+continue
+}
+objectProperties = append(objectProperties, fmt.Sprintf("%s %s", snakeCaseToCamelCase(name), getGolangTypeFromValue(object.Name, originalName, value)))
+}
+
 var requiredParamSetters []string
 for _, paramName := range getRequiredParamNames(object) {
 requiredParamSetters = append(requiredParamSetters, fmt.Sprintf("%s: %s", paramName, paramName))
@@ -209,14 +341,26 @@ func createConstraint(object objectJson) {
 
 var setters []string
 var objectProperties []string
+objectPropertyNames := make(map[string]bool)
 for _, param := range getRequiredParams(object) {
+objectPropertyNames[param[0]] = true
 objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
 }
 for _, param := range getOptionalParams(object) {
+objectPropertyNames[param[0]] = true
 objectProperties = append(objectProperties, fmt.Sprintf("%s %s", param[0], param[1]))
 setters = append(setters, getOptionString(object.Name, param[0], param[1]))
 }
 
+subConfig := object.Config["config"].(map[string]interface{})
+for originalName, value := range subConfig {
+name := snakeCaseToCamelCase(originalName)
+if _, ok := objectPropertyNames[name]; ok {
+continue
+}
+objectProperties = append(objectProperties, fmt.Sprintf("%s %s", snakeCaseToCamelCase(name), getGolangTypeFromValue(object.Name, originalName, value)))
+}
+
 var requiredParamSetters []string
 for _, paramName := range getRequiredParamNames(object) {
 requiredParamSetters = append(requiredParamSetters, fmt.Sprintf("%s: %s", paramName, paramName))

generate/python/generate_keras_objects.py

@@ -112,6 +112,14 @@
  {"type": "layer", "class": k.layers.experimental.EinsumDense, "args": []},
  {"type": "layer", "class": k.layers.experimental.RandomFourierFeatures, "args": []},
  {"type": "layer", "class": k.layers.experimental.SyncBatchNormalization, "args": []},
+ {"type": "optimizer", "class": k.optimizers.Adagrad, "args": []},
+ {"type": "optimizer", "class": k.optimizers.Adadelta, "args": []},
+ {"type": "optimizer", "class": k.optimizers.Adam, "args": []},
+ {"type": "optimizer", "class": k.optimizers.Adamax, "args": []},
+ {"type": "optimizer", "class": k.optimizers.Ftrl, "args": []},
+ {"type": "optimizer", "class": k.optimizers.Nadam, "args": []},
+ {"type": "optimizer", "class": k.optimizers.RMSprop, "args": []},
+ {"type": "optimizer", "class": k.optimizers.SGD, "args": []},
 ]
 defaults = {
  "activation": "linear",