diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index 3826dfe6..ffb777b5 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -5,7 +5,7 @@ name: Python application on: push: - branches: [ "main", "109-spelling-errors-in-license" ] + branches: [ "main", "115-add-use-examples-regnet-fine-tuning-on-cifar-10" ] permissions: contents: read @@ -24,6 +24,7 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip3 install -r cicd-requirements.txt - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -38,17 +39,20 @@ jobs: # run: python3 random_search.py # - name: Test CerebrosRealNeuronNetwork # run: python3 realnn-regression-example-ames-no-preproc.py - timeout-minutes: 45 - - name: Test distributed random search Ames by running - run: python3 regression-example-ames-no-preproc.py - - name: Test distributed random search Ames by running - Val set - run: python3 regression-example-ames-no-preproc-val-set.py - - name: Test text classifier - random search - ham-spam - run: python3 text-class-ham-or-spam.py - timeout-minutes: 90 - # - name: Test image classifier - small subset of CIFAR10 + # timeout-minutes: 45 + # - name: Test distributed random search Ames by running + # run: python3 regression-example-ames-no-preproc.py + # - name: Test distributed random search Ames by running - Val set + # run: python3 regression-example-ames-no-preproc-val-set.py + # - name: Test text classifier - random search - ham-spam + # run: python3 text-class-ham-or-spam.py # timeout-minutes: 90 - # run: python3 cifar10-example.py - - name: Test image classifier EfficientNetv2S - small subset of CIFAR10 + # # - name: Test image classifier - small subset of CIFAR10 + # # timeout-minutes: 90 + # # run: python3 cifar10-example.py + # - name: Test image classifier EfficientNetv2S - small subset of CIFAR10 + # timeout-minutes: 240 + # run: python3 cifar-10-efficientnetv2s.py + - name: Test image classifier RegNet timeout-minutes: 240 - run: python3 cifar-10-efficientnetv2s.py + run: python3 cifar_10_RegNet.py diff --git a/cicd-requirements.txt b/cicd-requirements.txt new file mode 100644 index 00000000..c3f91100 --- /dev/null +++ b/cicd-requirements.txt @@ -0,0 +1,2 @@ +tensorflow-datasets==4.9.3 +gmpy2==2.1.5 diff --git a/cifar_10_RegNet.ipynb b/cifar_10_RegNet.ipynb new file mode 100644 index 00000000..926cca31 --- /dev/null +++ b/cifar_10_RegNet.ipynb @@ -0,0 +1,3842 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "tags": [] + }, + "source": [ + "# Cifar-10 test for image classification" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "tags": [ + "imports" + ] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/sasha/jax-metal/lib/python3.8/site-packages/urllib3/__init__.py:34: NotOpenSSLWarning: urllib3 v2.0 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "import tensorflow as tf\n", + "import tensorflow_datasets as tfds\n", + "from tensorflow.keras.datasets import cifar10, cifar100\n", + "from tensorflow.keras.utils import to_categorical\n", + "from tensorflow.keras.models import Model\n", + "from tensorflow.keras.layers import Input, Resizing, Lambda, Flatten, Dense\n", + "import pandas as pd\n", + "import numpy as np\n", + "from cerebros.simplecerebrosrandomsearch.simple_cerebros_random_search\\\n", + " import SimpleCerebrosRandomSearch\n", + "import pendulum\n", + "from cerebros.units.units import DenseUnit\n", + "from cerebros.denseautomlstructuralcomponent.dense_automl_structural_component\\\n", + " import zero_7_exp_decay, zero_95_exp_decay, simple_sigmoid\n", + "from ast import literal_eval" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Download RegNetX016 with Imagenet weights (1000 classes)" + ] + }, + { + "cell_type": "code", + "execution_count": 596, + "metadata": {}, + "outputs": [], + "source": [ + "regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016',\n", + " include_top=True,\n", + " include_preprocessing=True,\n", + " weights='imagenet',\n", + " input_tensor=None,\n", + " input_shape=None,\n", + " pooling=None,\n", + " classes=1000,\n", + " classifier_activation='softmax'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 597, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"regnetx016\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " input_130 (InputLayer) [(None, 224, 224, 3)] 0 [] \n", + " \n", + " regnetx016_prestem_rescali (None, 224, 224, 3) 0 ['input_130[0][0]'] \n", + " ng (Rescaling) \n", + " \n", + " regnetx016_stem_conv (Conv (None, 112, 112, 32) 864 ['regnetx016_prestem_rescaling\n", + " 2D) [0][0]'] \n", + " \n", + " regnetx016_stem_bn (BatchN (None, 112, 112, 32) 128 ['regnetx016_stem_conv[0][0]']\n", + " ormalization) \n", + " \n", + " regnetx016_stem_relu (ReLU (None, 112, 112, 32) 0 ['regnetx016_stem_bn[0][0]'] \n", + " ) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 112, 112, 72) 2304 ['regnetx016_stem_relu[0][0]']\n", + " 0_conv_1x1_1 (Conv2D) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 112, 112, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 112, 112, 72) 0 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 15552 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 5184 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 2304 ['regnetx016_stem_relu[0][0]']\n", + " 0_skip_1x1 (Conv2D) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add_248 ( (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_0_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_0_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['tf.__operators__.add_248[0][\n", + " 0_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 5184 ['regnetx016_Stage_0_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 15552 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 5184 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 288 ['regnetx016_Stage_0_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_249 ( (None, 56, 56, 72) 0 ['regnetx016_Stage_0_XBlock_1_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_0_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_0_XBlock_ (None, 56, 56, 72) 0 ['tf.__operators__.add_249[0][\n", + " 1_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 56, 56, 168) 12096 ['regnetx016_Stage_0_XBlock_1_\n", + " 0_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 56, 56, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 56, 56, 168) 0 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 12096 ['regnetx016_Stage_0_XBlock_1_\n", + " 0_skip_1x1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add_250 ( (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_0_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_250[0][\n", + " 0_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_251 ( (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_1_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_251[0][\n", + " 1_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_1_\n", + " 2_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_2_\n", + " 2_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_252 ( (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_2_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_1_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_252[0][\n", + " 2_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_2_\n", + " 3_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 36288 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 28224 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 672 ['regnetx016_Stage_1_XBlock_3_\n", + " 3_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_253 ( (None, 28, 28, 168) 0 ['regnetx016_Stage_1_XBlock_3_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_1_XBlock_2_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_1_XBlock_ (None, 28, 28, 168) 0 ['tf.__operators__.add_253[0][\n", + " 3_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 28, 28, 408) 68544 ['regnetx016_Stage_1_XBlock_3_\n", + " 0_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 28, 28, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 28, 28, 408) 0 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 68544 ['regnetx016_Stage_1_XBlock_3_\n", + " 0_skip_1x1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add_254 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_0_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_254[0][\n", + " 0_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_255 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_1_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_255[0][\n", + " 1_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_1_\n", + " 2_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_2_\n", + " 2_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_256 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_2_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_1_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_256[0][\n", + " 2_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_2_\n", + " 3_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_3_\n", + " 3_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_257 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_3_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_2_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_257[0][\n", + " 3_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_3_\n", + " 4_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_4_\n", + " 4_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_258 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_4_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_3_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_258[0][\n", + " 4_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_4_\n", + " 5_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_5_\n", + " 5_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_259 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_5_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_4_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_259[0][\n", + " 5_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_5_\n", + " 6_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_6_\n", + " 6_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_260 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_6_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_5_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_260[0][\n", + " 6_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_6_\n", + " 7_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_7_\n", + " 7_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_261 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_7_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_6_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_261[0][\n", + " 7_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_7_\n", + " 8_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_8_\n", + " 8_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_262 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_8_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_7_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_262[0][\n", + " 8_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_8_\n", + " 9_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 88128 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 166464 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 1632 ['regnetx016_Stage_2_XBlock_9_\n", + " 9_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_263 ( (None, 14, 14, 408) 0 ['regnetx016_Stage_2_XBlock_9_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_2_XBlock_8_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_2_XBlock_ (None, 14, 14, 408) 0 ['tf.__operators__.add_263[0][\n", + " 9_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 14, 14, 912) 372096 ['regnetx016_Stage_2_XBlock_9_\n", + " 0_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 14, 14, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 14, 14, 912) 0 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 196992 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 831744 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 372096 ['regnetx016_Stage_2_XBlock_9_\n", + " 0_skip_1x1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_0_\n", + " 0_skip_bn (BatchNormalizat skip_1x1[0][0]'] \n", + " ion) \n", + " \n", + " tf.__operators__.add_264 ( (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_0_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_3_XBlock_0_\n", + " skip_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['tf.__operators__.add_264[0][\n", + " 0_exit_relu (ReLU) 0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 831744 ['regnetx016_Stage_3_XBlock_0_\n", + " 1_conv_1x1_1 (Conv2D) exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_1_bn (BatchNorm conv_1x1_1[0][0]'] \n", + " alization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_1_relu (ReLU) conv_1x1_1_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 196992 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_3x3 (Conv2D) conv_1x1_1_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_3x3_bn (BatchNormal conv_3x3[0][0]'] \n", + " ization) \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_3x3_relu (ReLU) conv_3x3_bn[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 831744 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_2 (Conv2D) conv_3x3_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 3648 ['regnetx016_Stage_3_XBlock_1_\n", + " 1_conv_1x1_2_bn (BatchNorm conv_1x1_2[0][0]'] \n", + " alization) \n", + " \n", + " tf.__operators__.add_265 ( (None, 7, 7, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " TFOpLambda) conv_1x1_2_bn[0][0]', \n", + " 'regnetx016_Stage_3_XBlock_0_\n", + " exit_relu[0][0]'] \n", + " \n", + " regnetx016_Stage_3_XBlock_ (None, 7, 7, 912) 0 ['tf.__operators__.add_265[0][\n", + " 1_exit_relu (ReLU) 0]'] \n", + " \n", + " 16_head_gap (GlobalAverage (None, 912) 0 ['regnetx016_Stage_3_XBlock_1_\n", + " Pooling2D) exit_relu[0][0]'] \n", + " \n", + " 16head_dense (Dense) (None, 1000) 913000 ['16_head_gap[0][0]'] \n", + " \n", + "==================================================================================================\n", + "Total params: 9233640 (35.22 MB)\n", + "Trainable params: 9190136 (35.06 MB)\n", + "Non-trainable params: 43504 (169.94 KB)\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "regnet.summary()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Make all layers untrainable except for the very last convolutional layer" + ] + }, + { + "cell_type": "code", + "execution_count": 606, + "metadata": {}, + "outputs": [], + "source": [ + "for layer in regnet.layers:\n", + " layer.trainable = False\n", + "regnet.layers[-6].trainable = True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Cifar-10 testing" + ] + }, + { + "cell_type": "code", + "execution_count": 607, + "metadata": {}, + "outputs": [], + "source": [ + "(X_train, y_train), (X_test, y_test) = cifar10.load_data()" + ] + }, + { + "cell_type": "code", + "execution_count": 608, + "metadata": {}, + "outputs": [], + "source": [ + "y_train_cat = to_categorical(y_train, 1000)\n", + "y_test_cat = to_categorical(y_test, 1000)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lambda layer for preprocessing" + ] + }, + { + "cell_type": "code", + "execution_count": 609, + "metadata": {}, + "outputs": [], + "source": [ + "def resize(x):\n", + " return tf.image.resize(x,size=(224,224),method='bicubic')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Modify the model" + ] + }, + { + "cell_type": "code", + "execution_count": 610, + "metadata": {}, + "outputs": [], + "source": [ + "input_shape = (32,32,3)" + ] + }, + { + "cell_type": "code", + "execution_count": 611, + "metadata": {}, + "outputs": [], + "source": [ + "input_layer = Input(shape=input_shape)\n", + "prep = Lambda(resize)(input_layer)\n", + "out = regnet(prep)\n", + "regnet_mod = Model(inputs=input_layer, outputs=out)" + ] + }, + { + "cell_type": "code", + "execution_count": 612, + "metadata": {}, + "outputs": [], + "source": [ + "regnet_mod.compile(optimizer='adam',\n", + " loss=tf.keras.losses.CategoricalCrossentropy(),\n", + " metrics=[tf.keras.metrics.TopKCategoricalAccuracy(k=1, name='top_1_categorical_accuracy')])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try to fit it on Cifar-10 data and then evaluate (there is no hope this is gonna work ...)" + ] + }, + { + "cell_type": "code", + "execution_count": 613, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1563/1563 [==============================] - 1521s 971ms/step - loss: 5.9514 - top_1_categorical_accuracy: 0.0000e+00\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 613, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "regnet_mod.fit(X_train, y_train_cat)" + ] + }, + { + "cell_type": "code", + "execution_count": 614, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "313/313 [==============================] - 409s 1s/step - loss: 8.6985 - top_1_categorical_accuracy: 0.0000e+00\n" + ] + }, + { + "data": { + "text/plain": [ + "[8.698457717895508, 0.0]" + ] + }, + "execution_count": 614, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "regnet_mod.evaluate(X_test, y_test_cat)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try the same with adding a Cerebros \"add-on\" network" + ] + }, + { + "cell_type": "code", + "execution_count": 617, + "metadata": {}, + "outputs": [], + "source": [ + "INPUT_SHAPES = [input_shape]\n", + "OUTPUT_SHAPES = [10]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use 10k-15k random samples from Cifar-10 to speed up the process" + ] + }, + { + "cell_type": "code", + "execution_count": 618, + "metadata": {}, + "outputs": [], + "source": [ + "num_samples = 15_000\n", + "rng = np.random.default_rng()\n", + "ind = rng.permutation(X_train.shape[0])[:num_samples]" + ] + }, + { + "cell_type": "code", + "execution_count": 619, + "metadata": {}, + "outputs": [], + "source": [ + "training_x = [tf.constant(X_train[ind,:,:,:])]\n", + "y_train_cat = to_categorical(y_train[ind], 10)\n", + "train_labels = [tf.constant(y_train_cat)]" + ] + }, + { + "cell_type": "code", + "execution_count": 620, + "metadata": {}, + "outputs": [], + "source": [ + "regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016',\n", + " include_top=True,\n", + " include_preprocessing=True,\n", + " weights='imagenet',\n", + " input_tensor=None,\n", + " input_shape=None,\n", + " pooling=None,\n", + " classes=1000,\n", + " classifier_activation='softmax'\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 622, + "metadata": {}, + "outputs": [], + "source": [ + "for layer in regnet.layers:\n", + " layer.trainable = False\n", + "regnet.layers[-6].trainable = True" + ] + }, + { + "cell_type": "code", + "execution_count": 623, + "metadata": {}, + "outputs": [], + "source": [ + "regnet_io = Model(inputs=regnet.layers[0].input,\n", + " outputs=regnet.layers[-2].output)" + ] + }, + { + "cell_type": "code", + "execution_count": 624, + "metadata": {}, + "outputs": [], + "source": [ + "input_layer = Input(shape=input_shape)\n", + "prep = Lambda(resize)(input_layer)\n", + "out = Flatten()(regnet_io(prep))\n", + "base_mod = Model(inputs=input_layer, outputs=out)" + ] + }, + { + "cell_type": "code", + "execution_count": 626, + "metadata": {}, + "outputs": [], + "source": [ + "activation = 'relu'\n", + "predecessor_level_connection_affinity_factor_first = 2.0\n", + "predecessor_level_connection_affinity_factor_main = 0.97\n", + "max_consecutive_lateral_connections = 5\n", + "p_lateral_connection = 0.97\n", + "num_lateral_connection_tries_per_unit = 2\n", + "learning_rate = 0.001\n", + "epochs = 10 # [1, 100]\n", + "batch_size = 20\n", + "maximum_levels = 4 # [3,7]\n", + "maximum_units_per_level = 7 # [2,10]\n", + "maximum_neurons_per_unit = 4 # [2,20]" + ] + }, + { + "cell_type": "code", + "execution_count": 627, + "metadata": {}, + "outputs": [], + "source": [ + "# Final training task\n", + "TIME = pendulum.now(tz='America/New_York').__str__()[:16]\\\n", + " .replace('T', '_')\\\n", + " .replace(':', '_')\\\n", + " .replace('-', '_')\n", + "#\n", + "PROJECT_NAME = f'{TIME}_cerebros_auto_ml_test_cifar10_regnet'\n", + "#\n", + "meta_trial_number = 42\n", + "#\n", + "cerebros_automl = SimpleCerebrosRandomSearch(\n", + " unit_type=DenseUnit,\n", + " input_shapes=INPUT_SHAPES,\n", + " output_shapes=OUTPUT_SHAPES,\n", + " training_data=training_x,\n", + " labels=train_labels,\n", + " validation_split=0.2,\n", + " direction='maximize',\n", + " metric_to_rank_by=\"val_top_1_categorical_accuracy\",\n", + " minimum_levels=2,\n", + " maximum_levels=maximum_levels,\n", + " minimum_units_per_level=1,\n", + " maximum_units_per_level=maximum_units_per_level,\n", + " minimum_neurons_per_unit=1,\n", + " maximum_neurons_per_unit=maximum_neurons_per_unit,\n", + " activation=activation,\n", + " final_activation='softmax',\n", + " number_of_architecture_moities_to_try=3,\n", + " number_of_tries_per_architecture_moity=2,\n", + " minimum_skip_connection_depth=1,\n", + " maximum_skip_connection_depth=7,\n", + " predecessor_level_connection_affinity_factor_first=predecessor_level_connection_affinity_factor_first,\n", + " predecessor_level_connection_affinity_factor_first_rounding_rule='ceil',\n", + " predecessor_level_connection_affinity_factor_main=predecessor_level_connection_affinity_factor_main,\n", + " predecessor_level_connection_affinity_factor_main_rounding_rule='ceil',\n", + " predecessor_level_connection_affinity_factor_decay_main=zero_7_exp_decay,\n", + " seed=8675309,\n", + " max_consecutive_lateral_connections=max_consecutive_lateral_connections,\n", + " gate_after_n_lateral_connections=3,\n", + " gate_activation_function=simple_sigmoid,\n", + " p_lateral_connection=p_lateral_connection,\n", + " p_lateral_connection_decay=zero_95_exp_decay,\n", + " num_lateral_connection_tries_per_unit=num_lateral_connection_tries_per_unit,\n", + " learning_rate=learning_rate,\n", + " loss=tf.keras.losses.CategoricalCrossentropy(),\n", + " metrics=[tf.keras.metrics.TopKCategoricalAccuracy(\n", + " k=1, name='top_1_categorical_accuracy')\n", + " ],\n", + " epochs=epochs,\n", + " project_name=f\"{PROJECT_NAME}_meta_{meta_trial_number}\",\n", + " model_graphs='model_graphs',\n", + " batch_size=batch_size,\n", + " meta_trial_number=meta_trial_number,\n", + " base_models=[base_mod])" + ] + }, + { + "cell_type": "code", + "execution_count": 628, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'1': }, {'3': }, {'1': }, {'2': }, {'4': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "{'2': }\n", + "{'4': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "{'2': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , ]\n", + "materialized_predecessor_units [, , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , , ]\n", + "materialized_predecessor_units [, , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_4 called\n", + "materialized network layers\n", + "[, , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_116 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[0][0]', \n", + " seUnit_0000000000000001_tr 'model_116[0][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3651) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3651) 14604 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3652 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3655) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3655) 14620 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7312 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3657) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_dns_[0][0\n", + " _0_4_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[0][0]', \n", + " seUnit_0000000000000001_tr 'model_116[0][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3657) 14628 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_4_cat_[0][0\n", + " _0_4_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14632 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_4_btn_[0][0\n", + " _0_4_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3649 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1858) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_3_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_116[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'model_116[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1858) 7432 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 18590 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8459890 (32.27 MB)\n", + "Trainable params: 930760 (3.55 MB)\n", + "Non-trainable params: 7529130 (28.72 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 994s 2s/step - loss: 0.4907 - top_1_categorical_accuracy: 0.8506 - val_loss: 0.3979 - val_top_1_categorical_accuracy: 0.8613\n", + "Epoch 2/10\n", + "600/600 [==============================] - 928s 2s/step - loss: 0.2852 - top_1_categorical_accuracy: 0.9047 - val_loss: 0.4354 - val_top_1_categorical_accuracy: 0.8700\n", + "Epoch 3/10\n", + "600/600 [==============================] - 799s 1s/step - loss: 0.2207 - top_1_categorical_accuracy: 0.9266 - val_loss: 0.4468 - val_top_1_categorical_accuracy: 0.8757\n", + "Epoch 4/10\n", + "600/600 [==============================] - 772s 1s/step - loss: 0.1676 - top_1_categorical_accuracy: 0.9400 - val_loss: 0.4882 - val_top_1_categorical_accuracy: 0.8803\n", + "Epoch 5/10\n", + "600/600 [==============================] - 748s 1s/step - loss: 0.1410 - top_1_categorical_accuracy: 0.9496 - val_loss: 0.4127 - val_top_1_categorical_accuracy: 0.8887\n", + "Epoch 6/10\n", + "600/600 [==============================] - 772s 1s/step - loss: 0.1055 - top_1_categorical_accuracy: 0.9638 - val_loss: 0.4178 - val_top_1_categorical_accuracy: 0.9043\n", + "Epoch 7/10\n", + "600/600 [==============================] - 762s 1s/step - loss: 0.0846 - top_1_categorical_accuracy: 0.9717 - val_loss: 0.5250 - val_top_1_categorical_accuracy: 0.8990\n", + "Epoch 8/10\n", + "600/600 [==============================] - 751s 1s/step - loss: 0.0880 - top_1_categorical_accuracy: 0.9674 - val_loss: 0.4680 - val_top_1_categorical_accuracy: 0.8987\n", + "Epoch 9/10\n", + "600/600 [==============================] - 759s 1s/step - loss: 0.0766 - top_1_categorical_accuracy: 0.9728 - val_loss: 0.5247 - val_top_1_categorical_accuracy: 0.8960\n", + "Epoch 10/10\n", + "600/600 [==============================] - 655s 1s/step - loss: 0.0715 - top_1_categorical_accuracy: 0.9769 - val_loss: 0.5141 - val_top_1_categorical_accuracy: 0.8937\n", + "this is neural_network_spec_file 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.490705 0.850583 0.397892 \n", + "1 0.285176 0.904750 0.435407 \n", + "2 0.220730 0.926583 0.446836 \n", + "3 0.167554 0.940000 0.488218 \n", + "4 0.141045 0.949583 0.412745 \n", + "5 0.105537 0.963750 0.417840 \n", + "6 0.084596 0.971667 0.525037 \n", + "7 0.088015 0.967417 0.468035 \n", + "8 0.076636 0.972833 0.524680 \n", + "9 0.071529 0.976917 0.514112 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.861333 0 0 \n", + "1 0.870000 0 0 \n", + "2 0.875667 0 0 \n", + "3 0.880333 0 0 \n", + "4 0.888667 0 0 \n", + "5 0.904333 0 0 \n", + "6 0.899000 0 0 \n", + "7 0.898667 0 0 \n", + "8 0.896000 0 0 \n", + "9 0.893667 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'1': }, {'3': }, {'1': }, {'2': }, {'4': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "{'2': }\n", + "{'4': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "{'2': }\n", + "{'4': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , , ]\n", + "materialized_predecessor_units [, , , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_4 called\n", + "materialized network layers\n", + "[, , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_116 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[1][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[1][0]', \n", + " seUnit_0000000000000001_tr 'model_116[1][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 14616 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3655 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3653) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3653) 14612 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7308 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[1][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[1][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[1][0]', \n", + " seUnit_0000000000000001_tr 'model_116[1][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3658) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_dns_[0][0\n", + " _0_4_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]', \n", + " 'model_116[1][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3658) 14632 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_4_cat_[0][0\n", + " _0_4_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3649 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14636 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_4_btn_[0][0\n", + " _0_4_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1861) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_2_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_4_dns_[0][0\n", + " ]', \n", + " 'model_116[1][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1861) 7444 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 18620 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8459943 (32.27 MB)\n", + "Trainable params: 930803 (3.55 MB)\n", + "Non-trainable params: 7529140 (28.72 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 1182s 2s/step - loss: 0.1760 - top_1_categorical_accuracy: 0.9299 - val_loss: 0.3438 - val_top_1_categorical_accuracy: 0.9050\n", + "Epoch 2/10\n", + "600/600 [==============================] - 816s 1s/step - loss: 0.0677 - top_1_categorical_accuracy: 0.9768 - val_loss: 0.4418 - val_top_1_categorical_accuracy: 0.8923\n", + "Epoch 3/10\n", + "600/600 [==============================] - 779s 1s/step - loss: 0.0581 - top_1_categorical_accuracy: 0.9797 - val_loss: 0.4520 - val_top_1_categorical_accuracy: 0.8973\n", + "Epoch 4/10\n", + "600/600 [==============================] - 654s 1s/step - loss: 0.0568 - top_1_categorical_accuracy: 0.9800 - val_loss: 0.4652 - val_top_1_categorical_accuracy: 0.9023\n", + "Epoch 5/10\n", + "600/600 [==============================] - 791s 1s/step - loss: 0.0388 - top_1_categorical_accuracy: 0.9860 - val_loss: 0.4700 - val_top_1_categorical_accuracy: 0.9140\n", + "Epoch 6/10\n", + "600/600 [==============================] - 826s 1s/step - loss: 0.0435 - top_1_categorical_accuracy: 0.9847 - val_loss: 0.4884 - val_top_1_categorical_accuracy: 0.8983\n", + "Epoch 7/10\n", + "600/600 [==============================] - 771s 1s/step - loss: 0.0432 - top_1_categorical_accuracy: 0.9847 - val_loss: 0.5836 - val_top_1_categorical_accuracy: 0.8947\n", + "Epoch 8/10\n", + "600/600 [==============================] - 658s 1s/step - loss: 0.0500 - top_1_categorical_accuracy: 0.9824 - val_loss: 0.4696 - val_top_1_categorical_accuracy: 0.9097\n", + "Epoch 9/10\n", + "600/600 [==============================] - 692s 1s/step - loss: 0.0366 - top_1_categorical_accuracy: 0.9865 - val_loss: 0.4700 - val_top_1_categorical_accuracy: 0.9153\n", + "Epoch 10/10\n", + "600/600 [==============================] - 674s 1s/step - loss: 0.0405 - top_1_categorical_accuracy: 0.9871 - val_loss: 0.4935 - val_top_1_categorical_accuracy: 0.9140\n", + "this is neural_network_spec_file 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.175978 0.929933 0.343752 \n", + "1 0.067747 0.976833 0.441750 \n", + "2 0.058143 0.979750 0.451954 \n", + "3 0.056750 0.980000 0.465216 \n", + "4 0.038794 0.986000 0.470013 \n", + "5 0.043505 0.984667 0.488350 \n", + "6 0.043237 0.984667 0.583564 \n", + "7 0.049963 0.982417 0.469586 \n", + "8 0.036560 0.986500 0.470019 \n", + "9 0.040510 0.987083 0.493530 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.905000 0 1 \n", + "1 0.892333 0 1 \n", + "2 0.897333 0 1 \n", + "3 0.902333 0 1 \n", + "4 0.914000 0 1 \n", + "5 0.898333 0 1 \n", + "6 0.894667 0 1 \n", + "7 0.909667 0 1 \n", + "8 0.915333 0 1 \n", + "9 0.914000 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'4': }, {'1': }, {'1': }, {'2': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'4': }\n", + "{'1': }\n", + "{'1': }\n", + "{'2': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'4': }\n", + "{'1': }\n", + "{'1': }\n", + "{'2': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , , ]\n", + "materialized_predecessor_units [, , , , , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_116 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[2][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[2][0]', \n", + " seUnit_0000000000000001_tr 'model_116[2][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3649 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3650) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[2][0]', \n", + " 'model_116[2][0]', \n", + " 'model_116[2][0]', \n", + " 'model_116[2][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3650) 14600 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3651 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[2][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[2][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[2][0]', \n", + " seUnit_0000000000000001_tr 'model_116[2][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3651) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[2][0]', \n", + " 'model_116[2][0]', \n", + " 'model_116[2][0]', \n", + " 'model_116[2][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3651) 14604 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14596 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7304 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1855) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_2_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'model_116[2][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1855) 7420 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 18560 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8434208 (32.17 MB)\n", + "Trainable params: 912408 (3.48 MB)\n", + "Non-trainable params: 7521800 (28.69 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 640s 1s/step - loss: 0.1383 - top_1_categorical_accuracy: 0.9459 - val_loss: 0.3667 - val_top_1_categorical_accuracy: 0.9043\n", + "Epoch 2/10\n", + "600/600 [==============================] - 643s 1s/step - loss: 0.0395 - top_1_categorical_accuracy: 0.9860 - val_loss: 0.3484 - val_top_1_categorical_accuracy: 0.9093\n", + "Epoch 3/10\n", + "600/600 [==============================] - 654s 1s/step - loss: 0.0187 - top_1_categorical_accuracy: 0.9948 - val_loss: 0.3917 - val_top_1_categorical_accuracy: 0.9103\n", + "Epoch 4/10\n", + "600/600 [==============================] - 655s 1s/step - loss: 0.0332 - top_1_categorical_accuracy: 0.9887 - val_loss: 0.4562 - val_top_1_categorical_accuracy: 0.8993\n", + "Epoch 5/10\n", + "600/600 [==============================] - 502s 836ms/step - loss: 0.0347 - top_1_categorical_accuracy: 0.9880 - val_loss: 0.4185 - val_top_1_categorical_accuracy: 0.9197\n", + "Epoch 6/10\n", + "600/600 [==============================] - 638s 1s/step - loss: 0.0309 - top_1_categorical_accuracy: 0.9893 - val_loss: 0.4839 - val_top_1_categorical_accuracy: 0.9097\n", + "Epoch 7/10\n", + "600/600 [==============================] - 633s 1s/step - loss: 0.0287 - top_1_categorical_accuracy: 0.9902 - val_loss: 0.5381 - val_top_1_categorical_accuracy: 0.9017\n", + "Epoch 8/10\n", + "600/600 [==============================] - 629s 1s/step - loss: 0.0278 - top_1_categorical_accuracy: 0.9907 - val_loss: 0.6015 - val_top_1_categorical_accuracy: 0.8967\n", + "Epoch 9/10\n", + "600/600 [==============================] - 631s 1s/step - loss: 0.0296 - top_1_categorical_accuracy: 0.9902 - val_loss: 0.5621 - val_top_1_categorical_accuracy: 0.9060\n", + "Epoch 10/10\n", + "600/600 [==============================] - 640s 1s/step - loss: 0.0300 - top_1_categorical_accuracy: 0.9902 - val_loss: 0.5124 - val_top_1_categorical_accuracy: 0.9133\n", + "this is neural_network_spec_file 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.138277 0.945867 0.366690 \n", + "1 0.039526 0.986000 0.348371 \n", + "2 0.018677 0.994750 0.391696 \n", + "3 0.033219 0.988750 0.456189 \n", + "4 0.034746 0.988000 0.418502 \n", + "5 0.030941 0.989250 0.483875 \n", + "6 0.028671 0.990167 0.538074 \n", + "7 0.027793 0.990667 0.601478 \n", + "8 0.029634 0.990250 0.562078 \n", + "9 0.030023 0.990167 0.512427 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.904333 0 0 \n", + "1 0.909333 0 0 \n", + "2 0.910333 0 0 \n", + "3 0.899333 0 0 \n", + "4 0.919667 0 0 \n", + "5 0.909667 0 0 \n", + "6 0.901667 0 0 \n", + "7 0.896667 0 0 \n", + "8 0.906000 0 0 \n", + "9 0.913333 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'4': }, {'1': }, {'1': }, {'2': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'4': }\n", + "{'1': }\n", + "{'1': }\n", + "{'2': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'4': }\n", + "{'1': }\n", + "{'1': }\n", + "{'2': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_3 called\n", + "materialized network layers\n", + "[, , , , , , ]\n", + "materialized_predecessor_units [, , , , , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_116 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[3][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[3][0]', \n", + " seUnit_0000000000000001_tr 'model_116[3][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3649 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3650) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[3][0]', \n", + " 'model_116[3][0]', \n", + " 'model_116[3][0]', \n", + " 'model_116[3][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3650) 14600 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3651 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3651) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_dns_[0][0\n", + " _0_3_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[3][0]', \n", + " 'model_116[3][0]', \n", + " 'model_116[3][0]', \n", + " 'model_116[3][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[3][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[3][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[3][0]', \n", + " seUnit_0000000000000001_tr 'model_116[3][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3651) 14604 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_cat_[0][0\n", + " _0_3_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 2) 7304 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_3_btn_[0][0\n", + " _0_3_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 4) 14596 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1851) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_3_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_116[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_116[3][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_3_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1851) 7404 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 18520 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8434152 (32.17 MB)\n", + "Trainable params: 912360 (3.48 MB)\n", + "Non-trainable params: 7521792 (28.69 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 653s 1s/step - loss: 0.1151 - top_1_categorical_accuracy: 0.9543 - val_loss: 0.2917 - val_top_1_categorical_accuracy: 0.9173\n", + "Epoch 2/10\n", + "600/600 [==============================] - 659s 1s/step - loss: 0.0198 - top_1_categorical_accuracy: 0.9949 - val_loss: 0.3359 - val_top_1_categorical_accuracy: 0.9183\n", + "Epoch 3/10\n", + "600/600 [==============================] - 674s 1s/step - loss: 0.0214 - top_1_categorical_accuracy: 0.9940 - val_loss: 0.5640 - val_top_1_categorical_accuracy: 0.8947\n", + "Epoch 4/10\n", + "600/600 [==============================] - 644s 1s/step - loss: 0.0252 - top_1_categorical_accuracy: 0.9918 - val_loss: 0.4489 - val_top_1_categorical_accuracy: 0.9073\n", + "Epoch 5/10\n", + "600/600 [==============================] - 637s 1s/step - loss: 0.0251 - top_1_categorical_accuracy: 0.9912 - val_loss: 0.4634 - val_top_1_categorical_accuracy: 0.9033\n", + "Epoch 6/10\n", + "600/600 [==============================] - 576s 960ms/step - loss: 0.0214 - top_1_categorical_accuracy: 0.9928 - val_loss: 0.5137 - val_top_1_categorical_accuracy: 0.9020\n", + "Epoch 7/10\n", + "600/600 [==============================] - 561s 934ms/step - loss: 0.0224 - top_1_categorical_accuracy: 0.9921 - val_loss: 0.5125 - val_top_1_categorical_accuracy: 0.9103\n", + "Epoch 8/10\n", + "600/600 [==============================] - 628s 1s/step - loss: 0.0202 - top_1_categorical_accuracy: 0.9927 - val_loss: 0.5039 - val_top_1_categorical_accuracy: 0.9083\n", + "Epoch 9/10\n", + "600/600 [==============================] - 647s 1s/step - loss: 0.0200 - top_1_categorical_accuracy: 0.9933 - val_loss: 0.5567 - val_top_1_categorical_accuracy: 0.9003\n", + "Epoch 10/10\n", + "600/600 [==============================] - 637s 1s/step - loss: 0.0216 - top_1_categorical_accuracy: 0.9923 - val_loss: 0.5413 - val_top_1_categorical_accuracy: 0.9117\n", + "this is neural_network_spec_file 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.115109 0.954333 0.291735 \n", + "1 0.019750 0.994917 0.335861 \n", + "2 0.021421 0.994000 0.564023 \n", + "3 0.025178 0.991750 0.448896 \n", + "4 0.025096 0.991167 0.463357 \n", + "5 0.021422 0.992833 0.513711 \n", + "6 0.022427 0.992083 0.512489 \n", + "7 0.020209 0.992750 0.503906 \n", + "8 0.020023 0.993250 0.556714 \n", + "9 0.021571 0.992333 0.541296 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.917333 0 1 \n", + "1 0.918333 0 1 \n", + "2 0.894667 0 1 \n", + "3 0.907333 0 1 \n", + "4 0.903333 0 1 \n", + "5 0.902000 0 1 \n", + "6 0.910333 0 1 \n", + "7 0.908333 0 1 \n", + "8 0.900333 0 1 \n", + "9 0.911667 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'1': }, {'3': }, {'1': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , ]\n", + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_116 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[4][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[4][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[4][0]', \n", + " seUnit_0000000000000001_tr 'model_116[4][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[4][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[4][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[4][0]', \n", + " seUnit_0000000000000001_tr 'model_116[4][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[4][0]', \n", + " 'model_116[4][0]', \n", + " 'model_116[4][0]', \n", + " 'model_116[4][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 14616 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3649 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3655 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1842) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_116[4][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[4][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1842) 7368 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 18430 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8408489 (32.08 MB)\n", + "Trainable params: 894009 (3.41 MB)\n", + "Non-trainable params: 7514480 (28.67 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 658s 1s/step - loss: 0.1041 - top_1_categorical_accuracy: 0.9579 - val_loss: 0.2875 - val_top_1_categorical_accuracy: 0.9183\n", + "Epoch 2/10\n", + "600/600 [==============================] - 651s 1s/step - loss: 0.0176 - top_1_categorical_accuracy: 0.9960 - val_loss: 0.3231 - val_top_1_categorical_accuracy: 0.9237\n", + "Epoch 3/10\n", + "600/600 [==============================] - 650s 1s/step - loss: 0.0112 - top_1_categorical_accuracy: 0.9975 - val_loss: 0.3313 - val_top_1_categorical_accuracy: 0.9207\n", + "Epoch 4/10\n", + "600/600 [==============================] - 646s 1s/step - loss: 0.0127 - top_1_categorical_accuracy: 0.9958 - val_loss: 0.4187 - val_top_1_categorical_accuracy: 0.9087\n", + "Epoch 5/10\n", + "600/600 [==============================] - 642s 1s/step - loss: 0.0136 - top_1_categorical_accuracy: 0.9958 - val_loss: 0.4513 - val_top_1_categorical_accuracy: 0.9130\n", + "Epoch 6/10\n", + "600/600 [==============================] - 641s 1s/step - loss: 0.0194 - top_1_categorical_accuracy: 0.9933 - val_loss: 0.4801 - val_top_1_categorical_accuracy: 0.9170\n", + "Epoch 7/10\n", + "600/600 [==============================] - 630s 1s/step - loss: 0.0129 - top_1_categorical_accuracy: 0.9960 - val_loss: 0.4732 - val_top_1_categorical_accuracy: 0.9190\n", + "Epoch 8/10\n", + "600/600 [==============================] - 483s 804ms/step - loss: 0.0146 - top_1_categorical_accuracy: 0.9951 - val_loss: 0.5282 - val_top_1_categorical_accuracy: 0.9117\n", + "Epoch 9/10\n", + "600/600 [==============================] - 635s 1s/step - loss: 0.0179 - top_1_categorical_accuracy: 0.9937 - val_loss: 0.5243 - val_top_1_categorical_accuracy: 0.9040\n", + "Epoch 10/10\n", + "600/600 [==============================] - 665s 1s/step - loss: 0.0152 - top_1_categorical_accuracy: 0.9955 - val_loss: 0.5798 - val_top_1_categorical_accuracy: 0.9103\n", + "this is neural_network_spec_file 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000000.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.104142 0.957933 0.287475 \n", + "1 0.017591 0.996000 0.323133 \n", + "2 0.011172 0.997500 0.331279 \n", + "3 0.012682 0.995833 0.418734 \n", + "4 0.013560 0.995750 0.451277 \n", + "5 0.019442 0.993333 0.480109 \n", + "6 0.012906 0.996000 0.473203 \n", + "7 0.014645 0.995083 0.528175 \n", + "8 0.017871 0.993750 0.524280 \n", + "9 0.015223 0.995500 0.579796 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.918333 0 0 \n", + "1 0.923667 0 0 \n", + "2 0.920667 0 0 \n", + "3 0.908667 0 0 \n", + "4 0.913000 0 0 \n", + "5 0.917000 0 0 \n", + "6 0.919000 0 0 \n", + "7 0.911667 0 0 \n", + "8 0.904000 0 0 \n", + "9 0.910333 0 0 \n", + "\n", + " model_name \n", + "0 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "SimpleCerebrosRandomSearch.input_shapes: [(32, 32, 3)]\n", + "nan\n", + ">nnf>ceil\n", + "k is: 0 value is: [{'1': }]\n", + "0\n", + "k is: 1 value is: [{'1': }, {'3': }, {'1': }]\n", + "1\n", + "Trying to create level 1\n", + "We think level 1's predecessors are: [0]\n", + "k is: 2 value is: [{'10': }]\n", + "2\n", + "Trying to create Final level 2\n", + "Trying to create level 2\n", + "We think level final level 2's predecessors are: [0, 1]\n", + "levels:\n", + "[0, 1, 2]\n", + "{'0': 'InputUnitModule'}\n", + "InputLevel.input_shapes [(32, 32, 3)]\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "Debug: I am 2 selecting 1\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Setting levels_unmaterialized[0] level_number 0 to have first successor: levels_unmaterialized[:1], having level_numbers of [1, 2]\n", + "Setting levels_unmaterialized[1] level_number 1 to have first successor: levels_unmaterialized[:2], having level_numbers of [2]\n", + "Debug: successor_connectivity_errors_2d []\n", + "$$$$$$>>>>> Base model: \n", + "InputUnit.input_shape: (32, 32, 3)\n", + "{'1': }\n", + "{'3': }\n", + "{'1': }\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_0 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_1 called\n", + "materialized network layers\n", + "[, , , ]\n", + "materialized_predecessor_units [, , , ]\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_DenseLevel_0000000000000001_tr_0_DenseUnit_0000000000000001_tr_0_2 called\n", + "materialized network layers\n", + "[, , , , , ]\n", + "materialized_predecessor_units [, , , , , ]\n", + "{'10': }\n", + "debug: meta_level_number\n", + "Debug: successor_connectivity_errors_2d []\n", + "materialize:_NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0 called\n", + "materialized network layers\n", + "[, , , , , , , , , , , , , ]\n", + "materialized_predecessor_units [, , , , , , , , , , , , , ]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:At this time, the v2.11+ optimizer `tf.keras.optimizers.Adam` runs slowly on M1/M2 Macs, please use the legacy Keras optimizer instead, located at `tf.keras.optimizers.legacy.Adam`.\n", + "WARNING:absl:There is a known slowdown when using v2.11+ Keras optimizers on M1/M2 Macs. Falling back to the legacy Keras optimizer, i.e., `tf.keras.optimizers.legacy.Adam`.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 32, 32, 3), dtype=tf.float32, name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'), name='NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_InputLevel_0000000000000000_tr_0_InputUnit_0000000000000000_tr_0_0_inp'\")\n", + "\n", + "outputs\n", + "KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_/Softmax:0', description=\"created by layer 'NeuralNetworkFuture_0000000000000nan_tr_0_FinalDenseLevel_0000000000000002_tr_0_FinalDenseUnit_0000000000000002_tr_0_0_dns_'\")\n", + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_116 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[5][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[5][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[5][0]', \n", + " seUnit_0000000000000001_tr 'model_116[5][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[5][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[5][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[5][0]', \n", + " seUnit_0000000000000001_tr 'model_116[5][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[5][0]', \n", + " 'model_116[5][0]', \n", + " 'model_116[5][0]', \n", + " 'model_116[5][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 14616 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3649 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3655 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1848) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_1_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_116[5][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'model_116[5][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1848) 7392 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 18490 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8408573 (32.08 MB)\n", + "Trainable params: 894081 (3.41 MB)\n", + "Non-trainable params: 7514492 (28.67 MB)\n", + "__________________________________________________________________________________________________\n", + "None\n", + "Epoch 1/10\n", + "600/600 [==============================] - 644s 1s/step - loss: 0.1019 - top_1_categorical_accuracy: 0.9599 - val_loss: 0.2985 - val_top_1_categorical_accuracy: 0.9200\n", + "Epoch 2/10\n", + "600/600 [==============================] - 646s 1s/step - loss: 0.0134 - top_1_categorical_accuracy: 0.9971 - val_loss: 0.3565 - val_top_1_categorical_accuracy: 0.9173\n", + "Epoch 3/10\n", + "600/600 [==============================] - 646s 1s/step - loss: 0.0089 - top_1_categorical_accuracy: 0.9978 - val_loss: 0.3329 - val_top_1_categorical_accuracy: 0.9210\n", + "Epoch 4/10\n", + "600/600 [==============================] - 644s 1s/step - loss: 0.0108 - top_1_categorical_accuracy: 0.9968 - val_loss: 0.4518 - val_top_1_categorical_accuracy: 0.9067\n", + "Epoch 5/10\n", + "600/600 [==============================] - 656s 1s/step - loss: 0.0114 - top_1_categorical_accuracy: 0.9967 - val_loss: 0.4321 - val_top_1_categorical_accuracy: 0.9160\n", + "Epoch 6/10\n", + "600/600 [==============================] - 642s 1s/step - loss: 0.0103 - top_1_categorical_accuracy: 0.9973 - val_loss: 0.4458 - val_top_1_categorical_accuracy: 0.9177\n", + "Epoch 7/10\n", + "600/600 [==============================] - 669s 1s/step - loss: 0.0138 - top_1_categorical_accuracy: 0.9953 - val_loss: 0.5233 - val_top_1_categorical_accuracy: 0.9070\n", + "Epoch 8/10\n", + "600/600 [==============================] - 662s 1s/step - loss: 0.0158 - top_1_categorical_accuracy: 0.9953 - val_loss: 0.5360 - val_top_1_categorical_accuracy: 0.9070\n", + "Epoch 9/10\n", + "600/600 [==============================] - 535s 892ms/step - loss: 0.0208 - top_1_categorical_accuracy: 0.9926 - val_loss: 0.5190 - val_top_1_categorical_accuracy: 0.9103\n", + "Epoch 10/10\n", + "600/600 [==============================] - 624s 1s/step - loss: 0.0126 - top_1_categorical_accuracy: 0.9958 - val_loss: 0.4918 - val_top_1_categorical_accuracy: 0.9170\n", + "this is neural_network_spec_file 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/model_architectures/tr_0000000000000000_subtrial_0000000000000001.txt\n", + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000001/assets\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "returning trial 0 oracles\n", + " loss top_1_categorical_accuracy val_loss \\\n", + "0 0.101878 0.959933 0.298451 \n", + "1 0.013441 0.997083 0.356516 \n", + "2 0.008856 0.997833 0.332865 \n", + "3 0.010832 0.996833 0.451827 \n", + "4 0.011395 0.996667 0.432090 \n", + "5 0.010319 0.997250 0.445817 \n", + "6 0.013825 0.995333 0.523302 \n", + "7 0.015816 0.995333 0.535993 \n", + "8 0.020776 0.992583 0.519047 \n", + "9 0.012579 0.995750 0.491757 \n", + "\n", + " val_top_1_categorical_accuracy trial_number subtrial_number \\\n", + "0 0.920000 0 1 \n", + "1 0.917333 0 1 \n", + "2 0.921000 0 1 \n", + "3 0.906667 0 1 \n", + "4 0.916000 0 1 \n", + "5 0.917667 0 1 \n", + "6 0.907000 0 1 \n", + "7 0.907000 0 1 \n", + "8 0.910333 0 1 \n", + "9 0.917000 0 1 \n", + "\n", + " model_name \n", + "0 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "1 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "2 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "3 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "4 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "5 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "6 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "7 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "8 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "9 2023_10_21_15_39_cerebros_auto_ml_test_cifar10... \n", + "Index(['loss', 'top_1_categorical_accuracy', 'val_loss',\n", + " 'val_top_1_categorical_accuracy', 'trial_number', 'subtrial_number',\n", + " 'model_name'],\n", + " dtype='object')\n", + "metric_to_rank_by is: 'val_top_1_categorical_accuracy'\n", + "Type of metric_to_rank_by is: \n", + "Best result this trial was: 0.9236666560173036\n", + "Type of best result: \n", + "Best model name: 2023_10_21_15_39_cerebros_auto_ml_test_cifar10_meta_42/models/tr_0000000000000000_subtrial_0000000000000000\n" + ] + } + ], + "source": [ + "%%time\n", + "result = cerebros_automl.run_random_search()" + ] + }, + { + "cell_type": "code", + "execution_count": 630, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Best accuracy achieved is 0.9236666560173036\n", + "top-1 categorical accuracy\n" + ] + } + ], + "source": [ + "print(f'Best accuracy achieved is {result}')\n", + "print(f'top-1 categorical accuracy')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evaluating the best model found" + ] + }, + { + "cell_type": "code", + "execution_count": 632, + "metadata": {}, + "outputs": [], + "source": [ + "best_model_found = cerebros_automl.get_best_model()" + ] + }, + { + "cell_type": "code", + "execution_count": 633, + "metadata": {}, + "outputs": [], + "source": [ + "#\n", + "eval_loss = tf.keras.losses.CategoricalCrossentropy()\n", + "#\n", + "eval_metrics =\\\n", + "[tf.keras.metrics.TopKCategoricalAccuracy(k=1,\\\n", + " name='eval_top_1_categorical_accuracy'),\n", + " tf.keras.metrics.TopKCategoricalAccuracy(k=5,\\\n", + " name='eval_top_5_categorical_accuracy')\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 634, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model: \"NeuralNetworkFuture_0000000000000nan_tr_0_nn_materialized\"\n", + "__________________________________________________________________________________________________\n", + " Layer (type) Output Shape Param # Connected to \n", + "==================================================================================================\n", + " NeuralNetworkFuture_000000 [(None, 32, 32, 3)] 0 [] \n", + " 0000000nan_tr_0_InputLevel \n", + " _0000000000000000_tr_0_Inp \n", + " utUnit_0000000000000000_tr \n", + " _0_0_inp (InputLayer) \n", + " \n", + " model_116 (Functional) (None, 912) 8320640 ['NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_InputLevel_00000\n", + " 00000000000_tr_0_InputUnit_000\n", + " 0000000000000_tr_0_0_inp[0][0]\n", + " '] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[0][0]', \n", + " seUnit_0000000000000001_tr 'model_116[0][0]'] \n", + " _0_1_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_cat_[0][0\n", + " _0_1_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3) 10947 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_btn_[0][0\n", + " _0_1_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 0 ['model_116[0][0]', \n", + " 0000000nan_tr_0_DenseLevel 'model_116[0][0]', \n", + " _0000000000000001_tr_0_Den 'model_116[0][0]', \n", + " seUnit_0000000000000001_tr 'model_116[0][0]'] \n", + " _0_0_cat_ (Concatenate) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_1_dns_[0][0\n", + " _0_2_cat_ (Concatenate) ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]', \n", + " 'model_116[0][0]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3648) 14592 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_cat_[0][0\n", + " _0_0_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 3654) 14616 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_cat_[0][0\n", + " _0_2_btn_ (BatchNormalizat ]'] \n", + " ion) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3649 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_0_btn_[0][0\n", + " _0_0_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1) 3655 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_DenseLevel 00000nan_tr_0_DenseLevel_00000\n", + " _0000000000000001_tr_0_Den 00000000001_tr_0_DenseUnit_000\n", + " seUnit_0000000000000001_tr 0000000000001_tr_0_2_btn_[0][0\n", + " _0_2_dns_ (Dense) ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1842) 0 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_DenseLevel_00000\n", + " Level_0000000000000002_tr_ 00000000001_tr_0_DenseUnit_000\n", + " 0_FinalDenseUnit_000000000 0000000000001_tr_0_0_dns_[0][0\n", + " 0000002_tr_0_0_cat_ (Conca ]', \n", + " tenate) 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'model_116[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'model_116[0][0]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_1_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_0_dns_[0][0\n", + " ]', \n", + " 'NeuralNetworkFuture_00000000\n", + " 00000nan_tr_0_DenseLevel_00000\n", + " 00000000001_tr_0_DenseUnit_000\n", + " 0000000000001_tr_0_2_dns_[0][0\n", + " ]'] \n", + " \n", + " NeuralNetworkFuture_000000 (None, 1842) 7368 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_btn_ (Batch _cat_[0][0]'] \n", + " Normalization) \n", + " \n", + " NeuralNetworkFuture_000000 (None, 10) 18430 ['NeuralNetworkFuture_00000000\n", + " 0000000nan_tr_0_FinalDense 00000nan_tr_0_FinalDenseLevel_\n", + " Level_0000000000000002_tr_ 0000000000000002_tr_0_FinalDen\n", + " 0_FinalDenseUnit_000000000 seUnit_0000000000000002_tr_0_0\n", + " 0000002_tr_0_0_dns_ (Dense _btn_[0][0]'] \n", + " ) \n", + " \n", + "==================================================================================================\n", + "Total params: 8408489 (32.08 MB)\n", + "Trainable params: 894009 (3.41 MB)\n", + "Non-trainable params: 7514480 (28.67 MB)\n", + "__________________________________________________________________________________________________\n" + ] + } + ], + "source": [ + "best_model_found.compile(loss=eval_loss, metrics=eval_metrics)\n", + "best_model_found.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": 636, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluating best model found ...\n", + "Loss | Top-1 accuracy | Top-5 accuracy\n", + "313/313 [==============================] - 481s 2s/step - loss: 0.6344 - eval_top_1_categorical_accuracy: 0.8978 - eval_top_5_categorical_accuracy: 0.9945\n" + ] + }, + { + "data": { + "text/plain": [ + "[0.6343686580657959, 0.8978000283241272, 0.9944999814033508]" + ] + }, + "execution_count": 636, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"Evaluating best model found ...\")\n", + "print(\"Loss | Top-1 accuracy | Top-5 accuracy\")\n", + "y_test_cat = to_categorical(y_test, 10)\n", + "best_model_found.evaluate(X_test, y_test_cat)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "kubeflow_notebook": { + "autosnapshot": true, + "deploy_config": {}, + "docker_image": "", + "experiment_name": "image-classification-b", + "katib_metadata": { + "algorithm": { + "algorithmName": "bayesianoptimization", + "algorithmSettings": [ + { + "name": "random_state", + "value": "10" + }, + { + "name": "acq_optimizer", + "value": "auto" + }, + { + "name": "acq_func", + "value": "gp_hedge" + }, + { + "name": "base_estimator", + "value": "GP" + } + ] + }, + "maxFailedTrialCount": 10, + "maxTrialCount": 40, + "objective": { + "additionalMetricNames": [], + "goal": 1, + "objectiveMetricName": "val-top-1-categorical-accuracy", + "type": "maximize" + }, + "parallelTrialCount": 3, + "parameters": [ + { + "feasibleSpace": { + "list": [ + "relu", + "elu", + "gelu" + ] + }, + "name": "activation", + "parameterType": "categorical" + }, + { + "feasibleSpace": { + "max": "40", + "min": "1", + "step": "0.1" + }, + "name": "predecessor_level_connection_affinity_factor_first", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "65", + "min": "0.1", + "step": "0.1" + }, + "name": "predecessor_level_connection_affinity_factor_main", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "40", + "min": "2", + "step": "1" + }, + "name": "max_consecutive_lateral_connections", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "50", + "min": "0.1", + "step": "0.1" + }, + "name": "p_lateral_connection", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "40", + "min": "1", + "step": "1" + }, + "name": "num_lateral_connection_tries_per_unit", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "0.3", + "min": "0.00001", + "step": "0.00001" + }, + "name": "learning_rate", + "parameterType": "double" + }, + { + "feasibleSpace": { + "max": "7", + "min": "1", + "step": "1" + }, + "name": "epochs", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "40", + "min": "5", + "step": "1" + }, + "name": "batch_size", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "8", + "min": "2" + }, + "name": "maximum_levels", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "10", + "min": "2", + "step": "1" + }, + "name": "maximum_units_per_level", + "parameterType": "int" + }, + { + "feasibleSpace": { + "max": "10", + "min": "2", + "step": "1" + }, + "name": "maximum_neurons_per_unit", + "parameterType": "int" + } + ] + }, + "katib_run": true, + "pipeline_description": "Cifar 10 example", + "pipeline_name": "image-classification-b", + "snapshot_volumes": true, + "volume_access_mode": "rwm", + "volumes": [ + { + "annotations": [], + "mount_point": "/home/jovyan", + "name": "test-images-workspace-pmbn9", + "size": 5, + "size_type": "Gi", + "snapshot": false, + "type": "clone" + } + ] + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/cifar_10_RegNet.py b/cifar_10_RegNet.py new file mode 100644 index 00000000..434ee473 --- /dev/null +++ b/cifar_10_RegNet.py @@ -0,0 +1,221 @@ +# Initializing + +import tensorflow as tf +import tensorflow_datasets as tfds +from tensorflow.keras.datasets import cifar10, cifar100 +from tensorflow.keras.utils import to_categorical +from tensorflow.keras.models import Model +from tensorflow.keras.layers import Input, Resizing, Lambda, Flatten, Dense +import pandas as pd +import numpy as np +from cerebros.simplecerebrosrandomsearch.simple_cerebros_random_search\ + import SimpleCerebrosRandomSearch +import pendulum +from cerebros.units.units import DenseUnit +from cerebros.denseautomlstructuralcomponent.dense_automl_structural_component\ + import zero_7_exp_decay, zero_95_exp_decay, simple_sigmoid +from ast import literal_eval + +# Download RegNetX016 with Imagenet weights (1000 classes) + +regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016', + include_top=True, + include_preprocessing=True, + weights='imagenet', + input_tensor=None, + input_shape=None, + pooling=None, + classes=1000, + classifier_activation='softmax' + ) + +regnet.summary() + +# Make all layers untrainable except for the very last convolutional layer + +for layer in regnet.layers: + layer.trainable = False +regnet.layers[-6].trainable = True + +# Cifar-10 testing + +(X_train, y_train), (X_test, y_test) = cifar10.load_data() + +y_train_cat = to_categorical(y_train, 10) +y_test_cat = to_categorical(y_test, 10) + +# Lambda layer for preprocessing + +def resize(x): + return tf.image.resize(x,size=(224,224),method='bicubic') + +# Modify the model + +input_shape = (32,32,3) + +# input_layer = Input(shape=input_shape) +# prep = Lambda(resize)(input_layer) +# out = regnet(prep) +# regnet_mod = Model(inputs=input_layer, outputs=out) + +# regnet_mod.compile(optimizer='adam', +# loss=tf.keras.losses.CategoricalCrossentropy(), +# metrics=[tf.keras.metrics.TopKCategoricalAccuracy(k=1, name='top_1_categorical_accuracy')]) + +# # Try to fit it on Cifar-10 data and then evaluate (there is no hope this is gonna work ...) + +# regnet_mod.fit(X_train, y_train_cat) + +# regnet_mod.evaluate(X_test, y_test_cat) + +# Try the same with adding a Cerebros "add-on" network + +INPUT_SHAPES = [input_shape] +OUTPUT_SHAPES = [10] + +# Use 10k-15k random samples from Cifar-10 to speed up the process + +num_samples = 5_000 +rng = np.random.default_rng() +ind = rng.permutation(X_train.shape[0])[:num_samples] + +training_x = [tf.constant(X_train[ind,:,:,:])] +y_train_cat = to_categorical(y_train[ind], 10) +train_labels = [tf.constant(y_train_cat)] + +regnet = tf.keras.applications.regnet.RegNetX016(model_name='regnetx016', + include_top=True, + include_preprocessing=True, + weights='imagenet', + input_tensor=None, + input_shape=None, + pooling=None, + classes=1000, + classifier_activation='softmax' + ) + +for layer in regnet.layers: + layer.trainable = False +regnet.layers[-6].trainable = True + +regnet_io = Model(inputs=regnet.layers[0].input, + outputs=regnet.layers[-2].output) + +input_layer = Input(shape=input_shape) +prep = Lambda(resize)(input_layer) +out = Flatten()(regnet_io(prep)) +base_mod = Model(inputs=input_layer, outputs=out) + +# activation = 'relu' +# predecessor_level_connection_affinity_factor_first = 2.0 +# predecessor_level_connection_affinity_factor_main = 0.97 +# max_consecutive_lateral_connections = 5 +# p_lateral_connection = 0.97 +# num_lateral_connection_tries_per_unit = 2 +# learning_rate = 0.001 +# epochs = 10 # [1, 100] +# batch_size = 20 +# maximum_levels = 4 # [3,7] +# maximum_units_per_level = 7 # [2,10] +# maximum_neurons_per_unit = 4 # [2,20] + +# New + + +activation = 'elu' +predecessor_level_connection_affinity_factor_first = 40 +predecessor_level_connection_affinity_factor_main = 65 +max_consecutive_lateral_connections = 2 +p_lateral_connection = 0.22299 +num_lateral_connection_tries_per_unit = 1 +learning_rate = 0.000129686 +epochs = 7 # [1, 100] +batch_size = 27 +maximum_levels = 8 # [3,7] +maximum_units_per_level = 10 # [2,10] +maximum_neurons_per_unit = 2 # [2,20] + + +# Final training task +TIME = pendulum.now(tz='America/New_York').__str__()[:16]\ + .replace('T', '_')\ + .replace(':', '_')\ + .replace('-', '_') +# +PROJECT_NAME = f'{TIME}_cerebros_auto_ml_test_cifar10_regnet' +# +meta_trial_number = 42 +# +cerebros_automl = SimpleCerebrosRandomSearch( + unit_type=DenseUnit, + input_shapes=INPUT_SHAPES, + output_shapes=OUTPUT_SHAPES, + training_data=training_x, + labels=train_labels, + validation_split=0.2, + direction='maximize', + metric_to_rank_by="val_top_1_categorical_accuracy", + minimum_levels=2, + maximum_levels=maximum_levels, + minimum_units_per_level=1, + maximum_units_per_level=maximum_units_per_level, + minimum_neurons_per_unit=1, + maximum_neurons_per_unit=maximum_neurons_per_unit, + activation=activation, + final_activation='softmax', + number_of_architecture_moities_to_try=3, + number_of_tries_per_architecture_moity=2, + minimum_skip_connection_depth=1, + maximum_skip_connection_depth=7, + predecessor_level_connection_affinity_factor_first=predecessor_level_connection_affinity_factor_first, + predecessor_level_connection_affinity_factor_first_rounding_rule='ceil', + predecessor_level_connection_affinity_factor_main=predecessor_level_connection_affinity_factor_main, + predecessor_level_connection_affinity_factor_main_rounding_rule='ceil', + predecessor_level_connection_affinity_factor_decay_main=zero_7_exp_decay, + seed=8675309, + max_consecutive_lateral_connections=max_consecutive_lateral_connections, + gate_after_n_lateral_connections=3, + gate_activation_function=simple_sigmoid, + p_lateral_connection=p_lateral_connection, + p_lateral_connection_decay=zero_95_exp_decay, + num_lateral_connection_tries_per_unit=num_lateral_connection_tries_per_unit, + learning_rate=learning_rate, + loss=tf.keras.losses.CategoricalCrossentropy(), + metrics=[tf.keras.metrics.TopKCategoricalAccuracy( + k=1, name='top_1_categorical_accuracy') + ], + epochs=epochs, + project_name=f"{PROJECT_NAME}_meta_{meta_trial_number}", + model_graphs='model_graphs', + batch_size=batch_size, + meta_trial_number=meta_trial_number, + base_models=[base_mod]) + +# Commented out IPython magic to ensure Python compatibility. +#%%time +result = cerebros_automl.run_random_search() + +print(f'Best accuracy achieved is {result}') +print(f'top-1 categorical accuracy') + +# Evaluating the best model found + +best_model_found = cerebros_automl.get_best_model() + +# +eval_loss = tf.keras.losses.CategoricalCrossentropy() +# +eval_metrics =\ +[tf.keras.metrics.TopKCategoricalAccuracy(k=1,\ + name='eval_top_1_categorical_accuracy'), + tf.keras.metrics.TopKCategoricalAccuracy(k=5,\ + name='eval_top_5_categorical_accuracy') +] + +best_model_found.compile(loss=eval_loss, metrics=eval_metrics) +best_model_found.summary() + +print("Evaluating best model found ...") +print("Loss | Top-1 accuracy | Top-5 accuracy") +y_test_cat = to_categorical(y_test, 10) +best_model_found.evaluate(X_test, y_test_cat)