Merge branch 'm-a-hall-resumable'

Author: braun-steven

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.2-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-all.zip

src/main/java/weka/classifiers/functions/Dl4jMlpClassifier.java

@@ -155,10 +155,15 @@ public class Dl4jMlpClassifier extends RandomizableClassifier
    * The number of epochs to perform.
    */
   protected int numEpochs = 10;
+  /** The current upper bound for the number of epochs */
+  protected int maxEpochs = 0;
   /**
-   * The number of epochs that have been performed.
+   * The total number of epochs that have been performed.
    */
   protected int numEpochsPerformed;
+
+  /** The number of epochs performed in this session of iterating */
+  protected int numEpochsPerformedThisSession;
   /**
    * The dataset trainIterator.
    */
@@ -211,6 +216,12 @@ public class Dl4jMlpClassifier extends RandomizableClassifier
    */
   protected LogConfiguration logConfig = new LogConfiguration();
 
+  /**
+   * Whether to allow training to continue at a later point after the initial
+   * model is built.
+   */
+  protected boolean resume;
+
 
   /**
    * Get the log configuration.
@@ -231,8 +242,7 @@ public LogConfiguration getLogConfig() {
           description = "The log configuration.",
           commandLineParamName = "logConfig",
           commandLineParamSynopsis = "-logConfig <LogConfiguration>",
-          displayOrder = 1
-  )
+          displayOrder = 1)
   public void setLogConfig(LogConfiguration logConfig) {
     this.logConfig = logConfig;
   }
@@ -617,6 +627,34 @@ public void setQueueSize(int QueueSize) {
     queueSize = QueueSize;
   }
 
+  /**
+   * If called with argument true, then the next time done() is called the model is effectively
+   * "frozen" and no further iterations can be performed
+   *
+   * @param resume true if the model is to be finalized after performing iterations
+   */
+
+  @OptionMetadata(description = "Set whether training can be resumed at a later date",
+    displayName = "Allow training to be resumed after the set number of epochs",
+    commandLineParamName = "resume",
+    commandLineParamSynopsis = "-resume",
+    commandLineParamIsFlag = true,
+    displayOrder = 31)
+  public void setResume(boolean resume) {
+    this.resume = resume;
+  }
+
+  /**
+   * Returns true if the model is to be finalized (or has been finalized) after
+   * training.
+   *
+   * @return the current value of finalize
+   */
+  public boolean getResume() {
+    return resume;
+  }
+
+
   /**
    * The method used to train the classifier.
    *
@@ -654,9 +692,20 @@ public void buildClassifier(Instances data) throws Exception {
   public void initializeClassifier(Instances data) throws Exception {
     logConfig.apply();
 
+    if (trainData != null && trainData.numInstances() > 0) {
+      // Resume run: only initialize iterator
+      trainIterator = getDataSetIterator(trainData);
+      return;
+    }
+
+    if (trainData != null) {
+      if (!trainData.equalHeaders(data)) {
+        throw new WekaException(trainData.equalHeadersMsg(data));
+      }
+    }
 
     // If only class is present, build zeroR
-    if (data.numAttributes() == 1 && data.classIndex() == 0) {
+    if (zeroR == null && data.numAttributes() == 1 && data.classIndex() == 0) {
       zeroR = new ZeroR();
       zeroR.buildClassifier(data);
       return;
@@ -698,11 +747,14 @@ public void initializeClassifier(Instances data) throws Exception {
     try {
       Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
 
-      // If zoo model was set, use this model as internal MultiLayerNetwork
-      if (useZooModel()) {
-        createZooModel();
-      } else {
-        createModel();
+      // Could be null due to resuming from a previous run
+      if (model == null) {
+        // If zoo model was set, use this model as internal MultiLayerNetwork
+        if (useZooModel()) {
+          createZooModel();
+        } else {
+          createModel();
+        }
       }
       // Initialize iterator
       instanceIterator.initialize();
@@ -715,11 +767,12 @@ public void initializeClassifier(Instances data) throws Exception {
         log.info(model.conf().toYaml());
       }
 
+      numEpochsPerformedThisSession = 0;
+      maxEpochs += numEpochs; // set the current upper bound
+
       // Set the iteration listener
       model.setListeners(getListener());
 
-      numEpochsPerformed = 0;
-
       isInitializationFinished = true;
     } finally {
       Thread.currentThread().setContextClassLoader(origLoader);
@@ -1201,10 +1254,12 @@ protected TrainingListener getListener() throws Exception {
 
     // Initialize weka listener
     if (iterationListener instanceof weka.dl4j.listener.EpochListener) {
-      int numEpochs = getNumEpochs();
+      // int numEpochs = getNumEpochs();
+      int numEpochs = maxEpochs;
       iterationListener
           .init(
               trainData.numClasses(),
+              numEpochsPerformed,
               numEpochs,
               numSamples,
               trainIterator,
@@ -1219,7 +1274,7 @@ protected TrainingListener getListener() throws Exception {
    */
   public boolean next() throws Exception {
 
-    if (numEpochsPerformed >= getNumEpochs() || zeroR != null || trainData == null) {
+    if (numEpochsPerformedThisSession >= getNumEpochs() || zeroR != null || trainData == null) {
       return false;
     }
 
@@ -1238,7 +1293,8 @@ public boolean next() throws Exception {
       trainIterator.reset();
       sw.stop();
       numEpochsPerformed++;
-      log.info("Epoch [{}/{}] took {}", numEpochsPerformed, numEpochs, sw.toString());
+      numEpochsPerformedThisSession++;
+      log.info("Epoch [{}/{}] took {}", numEpochsPerformed, maxEpochs, sw.toString());
     } finally {
       Thread.currentThread().setContextClassLoader(origLoader);
     }
@@ -1274,7 +1330,7 @@ public boolean useEarlyStopping() {
    */
   public void done() {
 
-    trainData = null;
+    trainData = new Instances(trainData,0);
   }
 
   /**

src/main/java/weka/classifiers/functions/RnnSequenceClassifier.java

@@ -37,14 +37,8 @@
 import org.nd4j.linalg.indexing.NDArrayIndex;
 import weka.classifiers.Classifier;
 import weka.classifiers.rules.ZeroR;
-import weka.core.Capabilities;
+import weka.core.*;
 import weka.core.Capabilities.Capability;
-import weka.core.CapabilitiesHandler;
-import weka.core.Instance;
-import weka.core.Instances;
-import weka.core.MissingOutputLayerException;
-import weka.core.OptionHandler;
-import weka.core.OptionMetadata;
 import weka.dl4j.CacheMode;
 import weka.dl4j.layers.Layer;
 import weka.dl4j.zoo.CustomNet;
@@ -84,6 +78,18 @@ public void initializeClassifier(Instances data) throws Exception {
     // Can classifier handle the data?
     getCapabilities().testWithFail(data);
 
+    if (trainData != null && trainData.numInstances() > 0) {
+      // Resume run: only initialize iterator
+      trainIterator = getDataSetIterator(trainData);
+      return;
+    }
+
+    if (trainData != null) {
+      if (!trainData.equalHeaders(data)) {
+        throw new WekaException(trainData.equalHeadersMsg(data));
+      }
+    }
+
     // Check basic network structure
     if (layers.length == 0) {
       throw new MissingOutputLayerException("No layers have been added!");
@@ -95,7 +101,7 @@ public void initializeClassifier(Instances data) throws Exception {
     }
 
     // If only class is present, build zeroR
-    if(data.numAttributes() == 1 && data.classIndex() == 0){
+    if(zeroR == null && data.numAttributes() == 1 && data.classIndex() == 0){
       zeroR = new ZeroR();
       zeroR.buildClassifier(data);
       return;
@@ -110,15 +116,20 @@ public void initializeClassifier(Instances data) throws Exception {
 
       instanceIterator.initialize();
 
-      createModel();
+      // Could be null due to resuming from a previous run
+      if (model == null){
+        createModel();
+      }
 
       // Setup the datasetiterators (needs to be done after the model initialization)
       trainIterator = getDataSetIterator(this.trainData);
 
       // Set the iteration listener
       model.setListeners(getListener());
 
-      numEpochsPerformed = 0;
+      numEpochsPerformedThisSession = 0;
+      maxEpochs += numEpochs; // set the current upper bound
+
     } finally {
       Thread.currentThread().setContextClassLoader(origLoader);
     }

src/main/java/weka/core/LogConfiguration.java

@@ -37,14 +37,15 @@
 import java.net.URI;
 import java.nio.file.Paths;
 import java.util.Collection;
+import java.util.Enumeration;
 
 /**
  * General logger configuration.
  *
  * @author Steven Lang
  */
 @Log4j2
-public class LogConfiguration implements Serializable {
+public class LogConfiguration implements Serializable, OptionHandler {
 
     private static final long serialVersionUID = 7910114399022582661L;
     /**
@@ -299,6 +300,34 @@ protected void updateLogLevel(String loggerName, LogLevel level) {
         Configurator.setLevel(loggerName, level.getLevel());
     }
 
+    /**
+     * Returns an enumeration describing the available options.
+     *
+     * @return an enumeration of all the available options
+     */
+    @Override public Enumeration<Option> listOptions() {
+        return Option.listOptionsForClass(this.getClass()).elements();
+    }
+
+    /**
+     * Parses a given list of options.
+     *
+     * @param options the list of options as an array of strings
+     * @throws Exception if an option is not supported
+     */
+    @Override public void setOptions(String[] options) throws Exception {
+        Option.setOptions(options, this, this.getClass());
+    }
+
+    /**
+     * Gets the current settings of the log configuration
+     *
+     * @return return an array of strings suitable for passing to setOptions
+     */
+    @Override public String[] getOptions() {
+        return Option.getOptions(this, this.getClass());
+    }
+
     /**
      * Available log levels.
      *

src/main/java/weka/dl4j/listener/EpochListener.java

@@ -44,7 +44,7 @@ public class EpochListener extends TrainingListener {
   private static final long serialVersionUID = -8852994767947925554L;
 
   /** Epoch counter */
-  private int currentEpoch = 0;
+  // private int currentEpoch = 0;
 
   /** Evaluate every N epochs */
   private int n = 5;

src/main/java/weka/dl4j/listener/TrainingListener.java

@@ -43,6 +43,8 @@ public abstract class TrainingListener
   protected int numClasses;
   /** Number of classes */
   protected int numEpochs;
+  /** The current epoch */
+  protected int currentEpoch;
   /** Training dataset iterator */
   protected transient DataSetIterator validationIterator;
   /** Validation dataset iterator */
@@ -58,11 +60,13 @@ public abstract class TrainingListener
    */
   public void init(
       int numClasses,
+      int currentEpoch,
       int numEpochs,
       int numSamples,
       DataSetIterator trainIterator,
       DataSetIterator validationIterator) {
     this.numClasses = numClasses;
+    this.currentEpoch = currentEpoch;
     this.numEpochs = numEpochs;
     this.numSamples = numSamples;
     this.trainIterator = trainIterator;