{ "cells": [ { "cell_type": "markdown", "id": "58768c00", "metadata": {}, "source": [ "# Forecasting moss & lichen fractional cover \n", "# **No weights** in the loss function\n", "## with a Neural Network using Keras\n", "## (Reading X and Y train/test files stored locally)\n", "### Using only 2m temperature and total precipitation from ERA5-land\n", "### For lichen output only" ] }, { "cell_type": "markdown", "id": "69c1b7c5", "metadata": {}, "source": [ "### This notebook uses TensorFlow NGC Container Release 23.03-tf2-py3\n", "### (https://catalog.ngc.nvidia.com/orgs/nvidia/containers/tensorflow)\n", "### and runs on a machine with 4x ARM CPUs (Neoverse N1) and 24GB RAM" ] }, { "cell_type": "code", "execution_count": 1, "id": "8e845957", "metadata": { "tags": [] }, "outputs": [], "source": [ "#pip install tables" ] }, { "cell_type": "code", "execution_count": 2, "id": "0b2d618c", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Starting imports\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2023-04-20 13:45:53.178927: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI FMA\n", "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", "2023-04-20 13:45:53.283831: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Finished imports\n" ] } ], "source": [ "print('Starting imports')\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import os\n", "import pandas as pd\n", "import tensorflow as tf\n", "from tensorflow.keras.layers import Input, BatchNormalization, Dense\n", "from tensorflow.keras.models import Model\n", "from tensorflow.keras.optimizers import Adam, SGD, Adadelta\n", "from tensorflow.keras import Sequential\n", "print('Finished imports')" ] }, { "cell_type": "code", "execution_count": 3, "id": "6aca9157", "metadata": { "tags": [] }, "outputs": [], "source": [ "# Sets all random seeds for the program to make it fully deterministic\n", "tf.keras.backend.experimental.enable_tf_random_generator()\n", "tf.keras.utils.set_random_seed(12345)" ] }, { "cell_type": "code", "execution_count": 4, "id": "1c58c9df-fb2d-4e03-8b6a-366b935a442a", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Depth = 1\n", "Width = 8\n", "Hidden layer activation = relu\n", "Final activation = sigmoid\n", "Learning rate = 0.01\n", "Epochs = 16\n", "Batch size = 128\n", "Loss function = Huber\n", "Validation split = 0.01\n", "Experiment name = TP1_365d_1_8_16epochs-128_relu-sigmoid_Adadelta-0.01-0.95_Huber_0.01split\n" ] } ], "source": [ "##### Defining various parameters\n", "\n", "depth = 1\n", "width = 8\n", "epochs = 16\n", "activation = 'relu'\n", "final_activation = 'sigmoid'\n", "batch_size = 128\n", "learning_rate = 0.01\n", "rho = 0.95\n", "loss = 'Huber'\n", "validation_split = 0.01\n", "print('Depth = ', depth)\n", "print('Width = ', width)\n", "print('Hidden layer activation = ', activation)\n", "print('Final activation = ', final_activation)\n", "print('Learning rate = ', learning_rate)\n", "print('Epochs = ', epochs)\n", "print('Batch size = ', batch_size)\n", "print('Loss function = ', loss)\n", "print('Validation split = ', validation_split)\n", "expname = 'TP1_365d_' + str(depth) + '_' + str(width) + '_' + str(epochs) + 'epochs-' + str(batch_size) + '_' + activation + '-' + final_activation + '_Adadelta-' + str(learning_rate) + '-' + str(rho) + '_' + loss + '_' + str(validation_split) + 'split'\n", "print('Experiment name = ', expname)" ] }, { "cell_type": "markdown", "id": "e011b533", "metadata": {}, "source": [ "# **All years** of input/output data - for training" ] }, { "cell_type": "code", "execution_count": 5, "id": "b641b6fb", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Reading X_train\n", "Reading X_test\n", "Reading y_train\n", "Reading y_test\n" ] } ], "source": [ "# Read local .hdf5 file\n", "path = '/home/jovyan/datahub/Reliance/ArcticBrowning/data/'\n", "\n", "print('Reading X_train')\n", "X_train_file = os.path.join(path, 'X_mean_tp1_train.hdf')\n", "input_train = pd.read_hdf(X_train_file)\n", "input_train.fillna(0, inplace=True)\n", "Ni_train = input_train['N']\n", "input_train = input_train.drop(columns=['N'])\n", "\n", "print('Reading X_test')\n", "X_test_file = os.path.join(path, 'X_mean_tp1_test.hdf')\n", "input_test = pd.read_hdf(X_test_file)\n", "input_test.fillna(0, inplace=True)\n", "Ni_test = input_test['N']\n", "input_test = input_test.drop(columns=['N'])\n", "\n", "print('Reading y_train')\n", "y_train_file = os.path.join(path, 'y_mean_tp1_train.hdf')\n", "output_train = pd.read_hdf(y_train_file)\n", "output_train.fillna(0, inplace=True)\n", "No_train = output_train['new_N']\n", "output_train = output_train.drop(columns=['new_N'])\n", "\n", "print('Reading y_test')\n", "y_test_file = os.path.join(path, 'y_mean_tp1_test.hdf')\n", "output_test = pd.read_hdf(y_test_file)\n", "output_test.fillna(0, inplace=True)\n", "No_test = output_test['new_N']\n", "output_test = output_test.drop(columns=['new_N'])" ] }, { "cell_type": "code", "execution_count": 6, "id": "75223e79", "metadata": {}, "outputs": [], "source": [ "train_loss_weights = (Ni_train + No_train) / 2." ] }, { "cell_type": "code", "execution_count": 7, "id": "c6272c2e", "metadata": {}, "outputs": [], "source": [ "test_loss_weights = (Ni_test + No_test) / 2." ] }, { "cell_type": "code", "execution_count": 8, "id": "df5cd5ac", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
| \n", " | Lichen | \n", "t2m_0 | \n", "t2m_1 | \n", "t2m_2 | \n", "t2m_3 | \n", "t2m_4 | \n", "t2m_5 | \n", "t2m_6 | \n", "t2m_7 | \n", "t2m_8 | \n", "... | \n", "tp_8750 | \n", "tp_8751 | \n", "tp_8752 | \n", "tp_8753 | \n", "tp_8754 | \n", "tp_8755 | \n", "tp_8756 | \n", "tp_8757 | \n", "tp_8758 | \n", "tp_8759 | \n", "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 37 | \n", "0.212308 | \n", "0.977626 | \n", "0.979172 | \n", "0.980195 | \n", "0.981094 | \n", "0.980679 | \n", "0.979638 | \n", "0.979272 | \n", "0.980954 | \n", "0.980093 | \n", "... | \n", "7.531988 | \n", "7.911303 | \n", "8.125458 | \n", "8.279312 | \n", "8.432280 | \n", "8.515192 | \n", "8.596331 | \n", "8.640226 | \n", "8.685230 | \n", "8.738436 | \n", "
| 3398 | \n", "0.123333 | \n", "0.969460 | \n", "0.969516 | \n", "0.969563 | \n", "0.968850 | \n", "0.967295 | \n", "0.963955 | \n", "0.961524 | \n", "0.967927 | \n", "0.970865 | \n", "... | \n", "0.138114 | \n", "0.140552 | \n", "0.141883 | \n", "0.145651 | \n", "0.155849 | \n", "0.181565 | \n", "0.188660 | \n", "0.196641 | \n", "0.198636 | \n", "0.199079 | \n", "
| 2747 | \n", "0.078287 | \n", "0.958717 | \n", "0.958901 | \n", "0.958894 | \n", "0.959793 | \n", "0.960436 | \n", "0.960108 | \n", "0.960259 | \n", "0.962879 | \n", "0.964013 | \n", "... | \n", "0.475308 | \n", "0.544697 | \n", "0.610540 | \n", "0.670618 | \n", "0.718503 | \n", "0.747767 | \n", "0.767719 | \n", "0.783016 | \n", "0.794766 | \n", "0.803855 | \n", "
| 2765 | \n", "0.122321 | \n", "0.975722 | \n", "0.975749 | \n", "0.976239 | \n", "0.976572 | \n", "0.977027 | \n", "0.977774 | \n", "0.978979 | \n", "0.983781 | \n", "0.985780 | \n", "... | \n", "3.759232 | \n", "4.259369 | \n", "4.789879 | \n", "5.254102 | \n", "5.741381 | \n", "6.185208 | \n", "6.641229 | \n", "7.126956 | \n", "7.510484 | \n", "7.827060 | \n", "
| 675 | \n", "0.186842 | \n", "0.993280 | \n", "0.992567 | \n", "0.991915 | \n", "0.991578 | \n", "0.991016 | \n", "0.990852 | \n", "0.989579 | \n", "0.988875 | \n", "0.987222 | \n", "... | \n", "0.992293 | \n", "1.706807 | \n", "2.269683 | \n", "2.554557 | \n", "2.763170 | \n", "3.015233 | \n", "3.313409 | \n", "3.566360 | \n", "3.778963 | \n", "4.003980 | \n", "
| ... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "... | \n", "
| 835 | \n", "0.099288 | \n", "0.968810 | \n", "0.969334 | \n", "0.969159 | \n", "0.968895 | \n", "0.969254 | \n", "0.970228 | \n", "0.971559 | \n", "0.976834 | \n", "0.976523 | \n", "... | \n", "0.927781 | \n", "0.927781 | \n", "0.927781 | \n", "0.927781 | \n", "0.927781 | \n", "0.927781 | \n", "0.928002 | \n", "0.928002 | \n", "0.928446 | \n", "0.928889 | \n", "
| 3264 | \n", "0.113205 | \n", "0.984804 | \n", "0.985838 | \n", "0.987426 | \n", "0.988205 | \n", "0.988538 | \n", "0.988480 | \n", "0.987478 | \n", "0.984979 | \n", "0.983080 | \n", "... | \n", "1.425258 | \n", "1.653823 | \n", "1.786394 | \n", "1.825856 | \n", "1.845364 | \n", "1.863543 | \n", "1.889259 | \n", "1.920961 | \n", "1.948230 | \n", "1.973059 | \n", "
| 1653 | \n", "0.119231 | \n", "0.958757 | \n", "0.955444 | \n", "0.953749 | \n", "0.954940 | \n", "0.956103 | \n", "0.956400 | \n", "0.957064 | \n", "0.960731 | \n", "0.960525 | \n", "... | \n", "0.727371 | \n", "0.921795 | \n", "1.152576 | \n", "1.407966 | \n", "1.744938 | \n", "2.252169 | \n", "2.924118 | \n", "3.832834 | \n", "4.896513 | \n", "5.866194 | \n", "
| 2607 | \n", "0.094717 | \n", "0.972968 | \n", "0.972829 | \n", "0.972340 | \n", "0.971712 | \n", "0.970634 | \n", "0.969718 | \n", "0.968187 | \n", "0.966529 | \n", "0.966880 | \n", "... | \n", "2.228670 | \n", "2.410679 | \n", "2.614414 | \n", "2.799084 | \n", "2.964909 | \n", "3.154456 | \n", "3.234487 | \n", "3.269736 | \n", "3.327376 | \n", "3.429133 | \n", "
| 2732 | \n", "0.129764 | \n", "0.971338 | \n", "0.971102 | \n", "0.970758 | \n", "0.970310 | \n", "0.968968 | \n", "0.967736 | \n", "0.967455 | \n", "0.970812 | \n", "0.972420 | \n", "... | \n", "2.058632 | \n", "2.226675 | \n", "2.419103 | \n", "2.611532 | \n", "2.840540 | \n", "2.973999 | \n", "3.071987 | \n", "3.226950 | \n", "3.300773 | \n", "3.415831 | \n", "
2722 rows × 17521 columns
\n", "| \n", " | new_Lichen | \n", "
|---|---|
| 989 | \n", "0.079881 | \n", "
| 686 | \n", "0.092609 | \n", "
| 472 | \n", "0.043143 | \n", "
| 3231 | \n", "0.082727 | \n", "
| 3351 | \n", "0.212667 | \n", "
| ... | \n", "... | \n", "
| 3268 | \n", "0.053333 | \n", "
| 1559 | \n", "0.030000 | \n", "
| 547 | \n", "0.000000 | \n", "
| 3251 | \n", "0.106951 | \n", "
| 228 | \n", "0.067881 | \n", "
681 rows × 1 columns
\n", "| \n", " | Actual_lichen | \n", "
|---|---|
| 0 | \n", "0.079881 | \n", "
| 1 | \n", "0.092609 | \n", "
| 2 | \n", "0.043143 | \n", "
| 3 | \n", "0.082727 | \n", "
| 4 | \n", "0.212667 | \n", "
| ... | \n", "... | \n", "
| 676 | \n", "0.053333 | \n", "
| 677 | \n", "0.030000 | \n", "
| 678 | \n", "0.000000 | \n", "
| 679 | \n", "0.106951 | \n", "
| 680 | \n", "0.067881 | \n", "
681 rows × 1 columns
\n", "