# trainer
sudo pip install keras
sudo pip install h5py
sudo pip install pyro4
sudo pip install redis
sudo pip install --upgrade pip enum34
# scheduler
sudo pip install zmq
# tests
sudo pip install pytest

Alternatively, use pip install -r src/requirements.txt to install all dependencies.

redis-server redis.conf
python -m Pyro4.naming # Start nameserver for pyro
python src/server.py <redis_port>
python src/client.py <cmd> <args...>

python src/client.py train <name> <config_file> <model_dir> <log_dir> <indices> <image_dir> <image_test_dir>

• name: Human readable name of your dataset.
• config_file: Path to config file. Example in config/example.
• model_dir: Path to output directory to store trained models.
• log_dir: Path to log directory.
• indices: Indices representing chokepoints for freezing layers. Should be inception, resnet, mobilenets or int repesenting a range
• image_dir: Path to directory of images to train on. Each subdirectory contains images for each class. Example of dataset in correct format on the Orca cluster at /datasets/BigLearning/ahjiang/image-data/training/flower_photos/
• image_test_dir: Path to test directory. (Optional)

Currently, models trained by M-Trainer are saved in .pb and .ckpt files. These need be merged into a single .pb file. This should be done programmatically but currently is done manually

python src/inference/freeze.py <model_prefix>

• model_prefix: Should be the name used to save checkpoint files. Files should exist called <model_prefix>.ckpt <model_prefix>.meta, and <model_prefix>.pb. Model will be saved as model_prefix-frozen.pb

cd /path/to/tensorflow
bazel build tensorflow/python/tools:freeze_graph
bazel-bin/tensorflow/python/tools/freeze_graph \
--input_graph=/users/ahjiang/models/saving_v2/flowers-1.pb  \
--input_binary=True \
--output_graph=/users/ahjiang/models/frozen_graph.pb \
--output_node_names dense_2/Softmax \
--input_checkpoint=/users/ahjiang/models/saving_v2/flowers-1.ckpt

# In Streamer. Run this server before sending schedules.
./apps/mainstream_server -C <config_dir> --camera <camera_name> -n <model_name> -m <model_dir>

• config_dir: Path to config dir.
• camera_name: Camera name as specified in <config_dir>/camera.toml
• model_name: Camera name as specified in <config_dir>/models.toml
• model_dir: Path to directory with model files (*.pb). Note: these should be frozen models.

video_desc = {"stream_fps": 30}
model_desc = {"total_layers": 41,
              "channels": 3,
              "height": 299,
              "width": 299,
              "layer_latencies": [1] * 41,
              "frozen_layer_names": {1: "input",
                                     10: "conv1",
                                     21: "conv2",
                                     30: "pool",
                                     40: "fc",
                                     41: "softmax"}}
apps = [
        {"app_id":1,
         "model_path": {
            0:  "/path/to/model.pb",
            10: "/path/to/model.pb",
            21: "/path/to/model.pb",
            30: "/path/to/model.pb",
            40: "/path/to/model.pb"
        },
        "event_length_ms": 10,
        "correlation": 0,
        "accuracies": {1: 1,
                       10: 0.8,
                       21: 0.6,
                       30: 0.6,
                       40: 0.2
                      }
        }]

schedule = Scheduler(apps, video_desc, model_desc)
schedule.run(cost_threshold)

• cost_threshold: Starting cost threshold (e.g., 250). More accurate starting cost threshold allows the scheduler to converge faster.

# In Mainstream, send schedules to Streamer
python src/scheduler/run_scheduler.py <num_apps_range> <version> <outfile_prefix>

• num_apps_range: Run Scheduler with 0 to num_apps_range applications. Applications specified in data/app_data.py.
• version: {0: Mainstream, 1: No Sharing, 2: Max Sharing}
• outfile_prefix: Path to outfile_prefix. Results will be written to outfile_prefix-{mainstream, nosharing, maxsharing}

cd /path/to/mainstream/
pytest -s test \
--tf_dir /path/to/tensorflow \
--data_dir /path/to/training/data

appendonly yes # Set appendonly on to allow persistence across redis instances