Tensorflow graph construction and training on building simulation data

import numpy as np
from besos import eppy_funcs as ef, sampling
from besos.evaluator import EvaluatorEP
from besos.problem import EPProblem
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from tensorflow.compat import v1 as tf

from parameter_sets import parameter_set

Couldn't import dot_parser, loading of dot files will not be possible.
WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
Instructions for updating:
non-resource variables are not supported in the long term

Generate data set

This generates an example model and sampling data, see this example.

parameters = parameter_set(7)
problem = EPProblem(parameters, ["Electricity:Facility"])
building = ef.get_building()
inputs = sampling.dist_sampler(sampling.lhs, problem, 30)
evaluator = EvaluatorEP(problem, building)
outputs = evaluator.df_apply(inputs)
results = inputs.join(outputs)
HBox(children=(FloatProgress(value=0.0, description='Executing', max=30.0, style=ProgressStyle(description_wid…
Conductivity Thickness U-Factor Solar Heat Gain Coefficient ElectricEquipment Lights Window to Wall Ratio Electricity:Facility
0 0.152149 0.161779 0.940276 0.442661 14.818443 14.964690 0.643468 2.347575e+09
1 0.084704 0.229743 1.246230 0.089919 11.174699 14.643703 0.860439 2.070557e+09
2 0.173219 0.116145 4.272470 0.045762 12.872851 12.463522 0.835218 2.045747e+09
3 0.127690 0.175980 2.267145 0.880006 10.282786 13.529331 0.626542 1.994944e+09
4 0.194943 0.283297 1.662965 0.200211 14.314024 10.283424 0.157865 2.020934e+09

Tensorflow Feed-forward Neural Network Example

1) Define Network Parameters

Static Parameters

All network parameters defined in this section are not part of the hyperparameter optimisation. Any of these parameters can be switched to an optimization parameter (see below).

learning_rate = 0.1
training_epochs = 4000
display_step = 300

n_hidden_1 = 10
n_hidden_2 = 10


Here we use the L2 norm regularization parameter alpha to calibrate the network bias-variance trade-off.

alpha = tf.placeholder(tf.float32, None, name="Alpha")
hy_par = [1e0, 1e1, 1e3]

2) Train-Test split, standardization

Next we split the data into a training set (80%) and a testing set (20%), and normalise it.

train_in, test_in, train_out, test_out = train_test_split(
    inputs, outputs, test_size=0.2
scaler = StandardScaler()
X_norm = scaler.fit_transform(X=train_in)
X_norm_test = scaler.transform(test_in)

scaler_out = StandardScaler()
y_norm = scaler_out.fit_transform(X=train_out)
y_norm_test = scaler_out.transform(test_out)

3) Set up the Tensorflow graph

Set up inputs and outputs as placeholder variables to be used in setting up the Tensorflow graph.

X = tf.placeholder(tf.float32, [None, len(X_norm[0, :])], name="X")
Y = tf.placeholder(tf.float32, [None, len(y_norm[0, :])], name="y")
with tf.name_scope("Variable_Definition"):
    weights = {
        "h1": tf.Variable(
            tf.random_normal([len(X_norm[0, :]), n_hidden_1]), name="HiddenLayer1"
        "h2": tf.Variable(
            tf.random_normal([n_hidden_1, n_hidden_2]), name="HiddenLayer2"
        "out": tf.Variable(
            tf.random_normal([n_hidden_2, len(y_norm[0, :])]), name="OutputLayer1"

    biases = {
        "b1": tf.Variable(tf.random_normal([n_hidden_1]), name="Bias"),
        "b2": tf.Variable(tf.random_normal([n_hidden_2]), name="Bias"),
        "out": tf.Variable(tf.random_normal([1])),
with tf.name_scope("FFNN_Model"):  # open up the Tensorflow name scope (context manager)

    def multilayer_perceptron(
    ):  # this function defines the Graph of our neural network
        with tf.name_scope("HL1"):
            layer_1 = tf.add(
                tf.matmul(x, weights["h1"]), biases["b1"]
            )  # apply Cartesian Product on inputs to the network (x) and the weights of layer 1, afterwards add the biases.
            layer_1 = tf.nn.relu(
            )  # Apply the relu activation function subsequently in each of the neurons
        with tf.name_scope("HL2"):
            layer_2 = tf.add(
                tf.matmul(layer_1, weights["h2"]), biases["b2"]
            )  # see above only we use layer_1 as input to layer_2
            layer_2 = tf.nn.relu(layer_2)
        with tf.name_scope("OutputLayer"):
            out_layer = tf.matmul(layer_2, weights["out"]) + biases["out"]
        return out_layer

    # 5b) Construct Model
    y_pred = multilayer_perceptron(X)
with tf.name_scope("Cost_regularized"):  # next element in the TF-Graph
    # set up mean squared error with L2 regularization
    loss_op = tf.reduce_mean(tf.square(Y - y_pred)) + alpha * (
        + tf.nn.l2_loss(weights["h2"])
        + tf.nn.l2_loss(weights["out"])
    tf.summary.scalar("Test", loss_op)  # observe the loss function throughout the run

R^2 score (operation definition)

with tf.name_scope("R2_Score"):
    total_error = tf.reduce_sum(tf.square(Y - tf.reduce_mean(Y)))
    unexplained_error = tf.reduce_sum(tf.square(Y - y_pred))
    R_squared = tf.subtract(1.0, tf.div(unexplained_error, total_error))
    tf.summary.scalar("R2", R_squared)
WARNING:tensorflow:From <ipython-input-10-7910d7f3ebbe>:4: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.

Mean Absolute Error (MAE)

with tf.name_scope("MAE"):
    mae = tf.reduce_mean(tf.abs(Y - y_pred))
    tf.summary.scalar("MAE", loss_op)
with tf.name_scope("Training"):
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, name="Training")
    train_op = optimizer.minimize(loss_op)
init_op = tf.global_variables_initializer()
def optimize(
    training_epochs, display_step, X_train, y_train, X_test, y_test, reg_par=None
    fig = plt.figure()
    for i in range(training_epochs):
        if reg_par == None:  # Check if hyperparameter provided
            sess.run([train_op, loss_op], feed_dict={X: X_train, Y: y_train})
            sess.run([train_op], feed_dict={X: X_train, Y: y_train, alpha: reg_par})

        if i % display_step == 0:
            # print(i)
            pred = sess.run(R_squared, feed_dict={X: X_test, Y: y_test})
            plt.plot(i, pred, "bx")
            pred = sess.run(R_squared, feed_dict={X: X_train, Y: y_train})
            plt.plot(i, pred, "rx")

            # create summary
            result = sess.run(
                merged, feed_dict={X: X_train, Y: y_train, alpha: reg_par}
            writer.add_summary(result, i)
            # plt.pause(0.1)
        "Finished! Accuracy of Network:",
        sess.run(R_squared, feed_dict={X: X_test, Y: y_test}),

Execute Tensorflow Graph

with tf.Session() as sess:
    merged = tf.summary.merge_all()
    writer = tf.summary.FileWriter(
        "logs/NN", sess.graph
    )  # for storing the neural networks

    hy_par_temp = hy_par[np.random.randint(0, len(hy_par))]
    print("Hyperparameter alpha: %.3f" % hy_par_temp)
    print("Training Network")
    # saver.save(sess=sess, save_path=get_save_path(i))
Hyperparameter alpha: 10.000
Training Network
Finished! Accuracy of Network: -0.02072835