Multiplatform training

Prep for training on aws boxes by ensuring num_cores is autodetected and that 
treelite can compile using correct compiler.

Author: cgreer

assess.py

@@ -102,5 +102,5 @@ def run_faceoff(
     run_generation_ladder(
         environment_name=sys.argv[1],
         species_list=species_list,
-        num_workers=settings.NUM_CORES,
+        num_workers=settings.ASSESSMENT_THREADS,
     )

gbdt_model.py

@@ -234,7 +234,7 @@ def train_from_training_data(
                 'num_leaves': num_leaves,
                 'max_bin': 128,
                 'min_data_in_leaf': min_data_in_leaf,
-                'num_threads': 16,  # 0 is as many as CPUs for server
+                'num_threads': settings.GBDT_TRAINING_THREADS,  # 0 is as many as CPUs for server
                 'verbose': 1,
                 # 'max_depth': 3,
                 # 'min_gain_to_split': 0.01,

paths.py

@@ -2,7 +2,7 @@
 from uuid import uuid4
 import pathlib
 
-from settings import ROOT_DATA_DIRECTORY, TMP_DIRECTORY
+from settings import ROOT_DATA_DIRECTORY, TMP_DIRECTORY, TOOL_CHAIN
 
 
 def full_path_mkdir_p(full_path):
@@ -62,7 +62,12 @@ def build_model_paths(environment, species, generation):
     # XXX VOMIT! Abstract this out.
     suffix = "model"
     if "gbdt" in species:
-        suffix = "dylib"
+        if TOOL_CHAIN == "clang":
+            suffix = "dylib"
+        elif TOOL_CHAIN == "gcc":
+            suffix = "so"
+        else:
+            raise KeyError(f"Unsure about suffix for tool chain: {TOOL_CHAIN}")
     base = build_model_directory(environment, species, generation)
     value_basename = f"value_model_{generation:06d}.{suffix}"
     policy_basename = f"policy_model_{generation:06d}.{suffix}"

settings.py

@@ -1,7 +1,10 @@
 import os
+import platform
+import psutil
 from pathlib import Path
 
 HOME = str(Path.home())
+OS_PLATFORM = platform.system()
 
 '''
 0
@@ -26,7 +29,23 @@
 MONITORING_DB_PATH = f"{HOME}/system_monitoring/monitoring.db"
 TMP_DIRECTORY = "/tmp"
 
-NUM_THREADS = 14
-NUM_CORES = 14
+NUM_CORES = psutil.cpu_count()
 
-TOOL_CHAIN = "clang" # Will need to change this for linux/windows
+SELF_PLAY_THREADS = NUM_CORES
+GBDT_TRAINING_THREADS = NUM_CORES
+TREELITE_THREADS = NUM_CORES
+ASSESSMENT_THREADS = NUM_CORES
+
+
+'''
+TOOL_CHAIN used by treelite to compile tree
+mac/os: clang
+linux: gcc
+windows: ?
+'''
+if OS_PLATFORM == "Darwin":
+    TOOL_CHAIN = "clang"
+elif OS_PLATFORM == "Linux":
+    TOOL_CHAIN = "gcc"
+else:
+    raise KeyError(f"Unhandled platform: {OS_PLATFORM}")

train_bot.py

@@ -1,6 +1,8 @@
 import math
 import time
 
+
+import settings
 from self_play import run as run_self_play
 from species import get_species
 from train import run as run_model_training
@@ -12,8 +14,8 @@ def run(
     environment,
     species_name,
     num_batches,
-    num_workers=12,
-    adjusted_win_rate_threshold=0.51,
+    num_workers=settings.SELF_PLAY_THREADS,
+    adjusted_win_rate_threshold=0.50,
     num_assessment_games=200,
 ):
     num_faceoff_rounds = math.ceil(num_assessment_games / num_workers) # Will play at least num_workers per round

treelite_model.py

@@ -58,7 +58,7 @@ def build_treelite_model(
     # Compile model to C/C++
     print("Compiling Tree")
     params = dict(
-        parallel_comp=settings.NUM_THREADS,
+        parallel_comp=settings.TREELITE_THREADS,
         # quantize=1, # Supposed to speed up predictions. Didn't when I tried it.
     )
     if annotation_results_path is not None: