Tensorflow graph construction and training on building simulation data¶

import numpy as np
from besos import eppy_funcs as ef, sampling
from besos.evaluator import EvaluatorEP
from besos.problem import EPProblem
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from tensorflow.compat import v1 as tf

from parameter_sets import parameter_set


tf.disable_v2_behavior()

Couldn't import dot_parser, loading of dot files will not be possible.
WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
Instructions for updating:
non-resource variables are not supported in the long term

Generate data set¶

This generates an example model and sampling data, see this example.

parameters = parameter_set(7)
problem = EPProblem(parameters, ["Electricity:Facility"])
building = ef.get_building()
inputs = sampling.dist_sampler(sampling.lhs, problem, 30)
evaluator = EvaluatorEP(problem, building)
outputs = evaluator.df_apply(inputs)
results = inputs.join(outputs)
results.head()

HBox(children=(FloatProgress(value=0.0, description='Executing', max=30.0, style=ProgressStyle(description_wid…

	Conductivity	Thickness	U-Factor	Solar Heat Gain Coefficient	ElectricEquipment	Lights	Window to Wall Ratio	Electricity:Facility
0	0.152149	0.161779	0.940276	0.442661	14.818443	14.964690	0.643468	2.347575e+09
1	0.084704	0.229743	1.246230	0.089919	11.174699	14.643703	0.860439	2.070557e+09
2	0.173219	0.116145	4.272470	0.045762	12.872851	12.463522	0.835218	2.045747e+09
3	0.127690	0.175980	2.267145	0.880006	10.282786	13.529331	0.626542	1.994944e+09
4	0.194943	0.283297	1.662965	0.200211	14.314024	10.283424	0.157865	2.020934e+09

Tensorflow Feed-forward Neural Network Example¶

1) Define Network Parameters¶

Static Parameters¶

All network parameters defined in this section are not part of the hyperparameter optimisation. Any of these parameters can be switched to an optimization parameter (see below).

learning_rate = 0.1
training_epochs = 4000
display_step = 300

n_hidden_1 = 10
n_hidden_2 = 10

Hyper-parameters¶

Here we use the L2 norm regularization parameter alpha to calibrate the network bias-variance trade-off.

alpha = tf.placeholder(tf.float32, None, name="Alpha")
hy_par = [1e0, 1e1, 1e3]

2) Train-Test split, standardization¶

Next we split the data into a training set (80%) and a testing set (20%), and normalise it.

train_in, test_in, train_out, test_out = train_test_split(
    inputs, outputs, test_size=0.2
)
scaler = StandardScaler()
X_norm = scaler.fit_transform(X=train_in)
X_norm_test = scaler.transform(test_in)

scaler_out = StandardScaler()
y_norm = scaler_out.fit_transform(X=train_out)
y_norm_test = scaler_out.transform(test_out)

3) Set up the Tensorflow graph¶

Set up inputs and outputs as placeholder variables to be used in setting up the Tensorflow graph.

X = tf.placeholder(tf.float32, [None, len(X_norm[0, :])], name="X")
Y = tf.placeholder(tf.float32, [None, len(y_norm[0, :])], name="y")

with tf.name_scope("Variable_Definition"):
    weights = {
        "h1": tf.Variable(
            tf.random_normal([len(X_norm[0, :]), n_hidden_1]), name="HiddenLayer1"
        ),
        "h2": tf.Variable(
            tf.random_normal([n_hidden_1, n_hidden_2]), name="HiddenLayer2"
        ),
        "out": tf.Variable(
            tf.random_normal([n_hidden_2, len(y_norm[0, :])]), name="OutputLayer1"
        ),
    }

    biases = {
        "b1": tf.Variable(tf.random_normal([n_hidden_1]), name="Bias"),
        "b2": tf.Variable(tf.random_normal([n_hidden_2]), name="Bias"),
        "out": tf.Variable(tf.random_normal([1])),
    }

with tf.name_scope("FFNN_Model"):  # open up the Tensorflow name scope (context manager)

    def multilayer_perceptron(
        x,
    ):  # this function defines the Graph of our neural network
        with tf.name_scope("HL1"):
            layer_1 = tf.add(
                tf.matmul(x, weights["h1"]), biases["b1"]
            )  # apply Cartesian Product on inputs to the network (x) and the weights of layer 1, afterwards add the biases.
            layer_1 = tf.nn.relu(
                layer_1
            )  # Apply the relu activation function subsequently in each of the neurons
        with tf.name_scope("HL2"):
            layer_2 = tf.add(
                tf.matmul(layer_1, weights["h2"]), biases["b2"]
            )  # see above only we use layer_1 as input to layer_2
            layer_2 = tf.nn.relu(layer_2)
        with tf.name_scope("OutputLayer"):
            out_layer = tf.matmul(layer_2, weights["out"]) + biases["out"]
        return out_layer

    # 5b) Construct Model
    y_pred = multilayer_perceptron(X)

with tf.name_scope("Cost_regularized"):  # next element in the TF-Graph
    # set up mean squared error with L2 regularization
    loss_op = tf.reduce_mean(tf.square(Y - y_pred)) + alpha * (
        tf.nn.l2_loss(weights["h1"])
        + tf.nn.l2_loss(weights["h2"])
        + tf.nn.l2_loss(weights["out"])
    )
    tf.summary.scalar("Test", loss_op)  # observe the loss function throughout the run

R^2 score (operation definition)

with tf.name_scope("R2_Score"):
    total_error = tf.reduce_sum(tf.square(Y - tf.reduce_mean(Y)))
    unexplained_error = tf.reduce_sum(tf.square(Y - y_pred))
    R_squared = tf.subtract(1.0, tf.div(unexplained_error, total_error))
    tf.summary.scalar("R2", R_squared)

WARNING:tensorflow:From <ipython-input-10-7910d7f3ebbe>:4: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Deprecated in favor of operator or tf.math.divide.

Mean Absolute Error (MAE)

with tf.name_scope("MAE"):
    mae = tf.reduce_mean(tf.abs(Y - y_pred))
    tf.summary.scalar("MAE", loss_op)

with tf.name_scope("Training"):
    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, name="Training")
    train_op = optimizer.minimize(loss_op)

init_op = tf.global_variables_initializer()

def optimize(
    training_epochs, display_step, X_train, y_train, X_test, y_test, reg_par=None
):
    sess.run(init_op)
    fig = plt.figure()
    for i in range(training_epochs):
        if reg_par == None:  # Check if hyperparameter provided
            sess.run([train_op, loss_op], feed_dict={X: X_train, Y: y_train})
        else:
            sess.run([train_op], feed_dict={X: X_train, Y: y_train, alpha: reg_par})

        if i % display_step == 0:
            # print(i)
            pred = sess.run(R_squared, feed_dict={X: X_test, Y: y_test})
            plt.plot(i, pred, "bx")
            pred = sess.run(R_squared, feed_dict={X: X_train, Y: y_train})
            plt.plot(i, pred, "rx")

            # create summary
            result = sess.run(
                merged, feed_dict={X: X_train, Y: y_train, alpha: reg_par}
            )
            writer.add_summary(result, i)
            # plt.pause(0.1)
    print(
        "Finished! Accuracy of Network:",
        sess.run(R_squared, feed_dict={X: X_test, Y: y_test}),
    )
    plt.close()
    return

Execute Tensorflow Graph¶

with tf.Session() as sess:
    merged = tf.summary.merge_all()
    writer = tf.summary.FileWriter(
        "logs/NN", sess.graph
    )  # for storing the neural networks

    hy_par_temp = hy_par[np.random.randint(0, len(hy_par))]
    print("Hyperparameter alpha: %.3f" % hy_par_temp)
    print("Training Network")
    optimize(
        training_epochs,
        display_step,
        X_norm,
        y_norm,
        X_norm_test,
        y_norm_test,
        reg_par=hy_par_temp,
    )
    # saver.save(sess=sess, save_path=get_save_path(i))

Hyperparameter alpha: 10.000
Training Network
Finished! Accuracy of Network: -0.02072835