3D Convolutional Neural Network input shape

Question

I'm having a problem feeding a 3D CNN using Keras and Python to classify 3D shapes. I have a folder with some models in JSON format. I read those models into a Numpy Array. The models are 25*25*25 and represent the occupancy grid of the voxelized model (each position represents if the voxel in position (i,j,k) has points in it or no), so I only have 1 channel of input, like grayscale images in 2D images. The code that I have is the following:

import numpy as np
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Convolution3D, MaxPooling3D
from keras.optimizers import SGD
from keras.utils import np_utils
from keras import backend as K

# Number of Classes and Epochs of Training
nb_classes = 3 # cube, cone or sphere
nb_epoch = 100
batch_size = 2

# Input Image Dimensions
img_rows, img_cols, img_depth = 25, 25, 25

# Number of Convolutional Filters to use
nb_filters = 32

# Convolution Kernel Size
kernel_size = [5,5,5]

X_train, Y_train = [], []

# Read from File
import os
import json

i=0
for filename in os.listdir(os.path.join(os.getcwd(), 'models')):
    with open(os.path.join(os.getcwd(), 'models', filename)) as f:
        file = f.readlines()
        json_file = '\n'.join(file)
        content = json.loads(json_file)
        occupancy = content['model']['occupancy']
        form = []
        for value in occupancy:
            form.append(int(value))
        final_model = [ [ [ 0 for i in range(img_rows) ]
                              for j in range(img_cols) ]
                              for k in range(img_depth) ]
        a = 0
        for i in range(img_rows):
            for j in range(img_cols):
                for k in range(img_depth):
                    final_model[i][j][k] = form[a]
                    a = a + 1
        X_train.append(final_model)
        Y_train.append(content['model']['label'])

X_train = np.array(X_train)
Y_train = np.array(Y_train)

# (1 channel, 25 rows, 25 cols, 25 of depth)
input_shape = (1, img_rows, img_cols, img_depth)

# Init
model = Sequential()

# 3D Convolution layer
model.add(Convolution3D(nb_filters, kernel_size[0], kernel_size[1], kernel_size[2],
                        input_shape=input_shape,
                        activation='relu'))

# Fully Connected layer
model.add(Flatten())
model.add(Dense(128,
          init='normal',
          activation='relu'))
model.add(Dropout(0.5))

# Softmax Layer
model.add(Dense(nb_classes,
                init='normal'))
model.add(Activation('softmax'))

# Compile
model.compile(loss='categorical_crossentropy',
              optimizer=SGD())

# Fit network
model.fit(X_train, Y_train, nb_epoch=nb_epoch,
         verbose=1)

After this, I get the following error

Using TensorFlow backend. Traceback (most recent call last): File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py", line 670, in _call_cpp_shape_fn_impl status) File "/usr/local/Cellar/python3/3.6.0/Frameworks/Python.framework/Versions/3.6/lib/python3.6/contextlib.py", line 89, in exit next(self.gen) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py", line 469, in raise_exception_on_not_ok_status pywrap_tensorflow.TF_GetCode(status)) tensorflow.python.framework.errors_impl.InvalidArgumentError: Negative dimension size caused by subtracting 5 from 1 for 'Conv3D' (op: 'Conv3D') with input shapes: [?,1,25,25,25], [5,5,5,25,32].

During handling of the above exception, another exception occurred:

Traceback (most recent call last): File "CNN_3D.py", line 76, in activation='relu')) File "/usr/local/lib/python3.6/site-packages/keras/models.py", line 299, in add layer.create_input_layer(batch_input_shape, input_dtype) File "/usr/local/lib/python3.6/site-packages/keras/engine/topology.py", line 401, in create_input_layer self(x) File "/usr/local/lib/python3.6/site-packages/keras/engine/topology.py", line 572, in call self.add_inbound_node(inbound_layers, node_indices, tensor_indices) File "/usr/local/lib/python3.6/site-packages/keras/engine/topology.py", line 635, in add_inbound_node Node.create_node(self, inbound_layers, node_indices, tensor_indices) File "/usr/local/lib/python3.6/site-packages/keras/engine/topology.py", line 166, in create_node output_tensors = to_list(outbound_layer.call(input_tensors[0], mask=input_masks[0])) File "/usr/local/lib/python3.6/site-packages/keras/layers/convolutional.py", line 1234, in call filter_shape=self.W_shape) File "/usr/local/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2831, in conv3d x = tf.nn.conv3d(x, kernel, strides, padding) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 522, in conv3d strides=strides, padding=padding, name=name) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 763, in apply_op op_def=op_def) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2397, in create_op set_shapes_for_outputs(ret) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1757, in set_shapes_for_outputs shapes = shape_func(op) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1707, in call_with_requiring return call_cpp_shape_fn(op, require_shape_fn=True) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py", line 610, in call_cpp_shape_fn debug_python_shape_fn, require_shape_fn) File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/common_shapes.py", line 675, in _call_cpp_shape_fn_impl raise ValueError(err.message) ValueError: Negative dimension size caused by subtracting 5 from 1 for 'Conv3D' (op: 'Conv3D') with input shapes: [?,1,25,25,25], [5,5,5,25,32].

What am I doing wrong to get this error?

Holland

I think the shape of your training data is wrong. Tensorflow expects the data to be in the form of (sample, dim1, dim2, ..., channel). Given a list of regular 2D images, you can reshape like this: X_train.reshape((-1, WIDTH, HEIGHT, 1)). Adapting this to your case you could try X_train = X_train.reshape((-1, img_rows, img_cols, img_depth, 1)). And the input_shape should be (img_rows, img_cols, img_depth, 1). – Isomerism

I still get the same error. I can pass the creation of the layers when I add more channels (input_shape = (5, img_rows, img_cols, img_depth)) that overcome or equal the size of the convolutional filters. But I only have a channel of input. I think that the problem is in the definition of the Conv3D layer – Enterprising

I am wanting to do exactly what you are doing, but I'm having trouble figuring out how to turn a 3D model file into an occupancy grid. How are you doing that? Thank you! – Sophy

Answer 1

I think that the problem is that you are setting the input shape in Theano ordering but you are using Keras with Tensorflow backend and Tensorflow img ordering. In addition the y_train array has to be converted to categorical labels.

Updated code:

from keras.utils import np_utils
from keras import backend as K

if K.image_dim_ordering() == 'th':
    X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols, img_depth)
    input_shape = (1, img_rows, img_cols, img_depth)
else:
    X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, img_depth, 1)
    input_shape = (img_rows, img_cols, img_depth, 1)

Y_train = np_utils.to_categorical(Y_train, nb_classes)

Adding this lines should fix it.

Curr

Using both codes almost had it. But now I get a new error: Using TensorFlow backend. Traceback (most recent call last): File "CNN_3D_2.py", line 86, in <module> verbose=1) ... ValueError: Error when checking model input: expected convolution3d_input_1 to have shape (None, 25, 25, 25, 1) but got array with shape (1, 25, 25, 25, 2) – Enterprising

can you tell me your original (before reshaping) X_train.shape ?? – Curr

(2, 25, 25, 25) – Enterprising

I updated whe code with the reshape @JoãoPedroFontes. Try now – Curr

David, it is giving the error ValueError: Input 0 is incompatible with layer convolution3d_1: expected ndim=5, found ndim=4. The example that you gave me is not for 2D? In the input shape you only give rows and columns. I tried adding the img_depths and it gave ValueError: Error when checking model target: expected activation_1 to have shape (None, 3) but got array with shape (2, 1) – Enterprising

@JoãoPedroFontes yes. It was a typo. Now is updated. The new error you are getting I assume that is now related to the y_train array. Could you tell me your y_train.shape? – Curr

I think it may have to be with the previous shape of the X_train array. It has (2, 25, 25, 25). I don't understand why it has that 2, because I only have 1 channel of input and 25*25*25 data – Enterprising

The Y_train array has (2,) shape – Enterprising

maybe you could share on the question your models file so we can reproduce the code – Curr

Okay @JoãoPedroFontes, You have to convert your y_train array. – Curr

here is an example of one of my files link – Enterprising

Answer 2

if you use 3d CNN to frame prediction ,you must use : input=( no. of frame ,hight, width ,channel ) my code def cnn_network(): model = Sequential()

# # Layer 1
# model.add(
# Conv3D(512, kernel_size=(3, 3, 3), strides=(1, 1, 1), input_shape=(6, 14, 14, 512), use_bias=512, padding='SAME',
#            activation='relu',
#            name='conv3D_1_1'))

# model.add(Conv3D(512, kernel_size=(3, 3, 3), strides=(1, 1, 1),use_bias=512, padding='SAME',
#        activation='relu',name='conv3D_1_5'))

# model.add(MaxPooling3D(pool_size=(4, 2, 2), strides=(1, 2, 2)))
# model.add(BatchNormalization())

# # Layer 2
# model.add(
#     Conv3DTranspose(512, kernel_size=(1, 3, 3), strides=(1, 2, 2), use_bias=512, padding='SAME', activation='relu',
#                     name='Deconv3D_16'))

# model.add(Conv3D(512, kernel_size=(3, 3, 3), strides=(1, 1, 1), use_bias=512,
#                  padding='SAME', activation='relu', name='conv3D_2__19'))

# #model.add(Conv3D(512, kernel_size=(3, 3, 3), strides=(1, 1, 1), use_bias=512,
#  #                padding='SAME', activation='relu', name='conv3D_2__77'))

# model.add(
# Conv3DTranspose(256, kernel_size=(3, 3, 3), strides=(1, 2, 2), use_bias=256, padding='SAME', activation='relu',
#                     name='Deconv3D_1'))

# model.add(Conv3D(256, kernel_size=(3, 3, 3), strides=(1, 1, 1), use_bias=256,
#                  padding='SAME', activation='relu', name='conv3D_2__1'))

# model.add(Conv3D(256, kernel_size=(3, 3, 3), strides=(1, 1, 1), use_bias=256, padding='SAME', activation='relu',
#                  name='conv3D_2__2'))
# model.add(MaxPooling3D(pool_size=(3, 1, 1), strides=(1, 1, 1)))
# #model.add(Conv3D(256, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=256, padding='SAME', activation='relu',
#                  #name='conv3D_18'))
# #model.add(MaxPooling3D(pool_size=(2, 1, 1), strides=(2, 1, 1)))
# model.add(BatchNormalization())
# # Layer 3

# model.add(
#     Conv3DTranspose(128, kernel_size=(1, 3, 3), strides=(1, 2, 2), use_bias=128, padding='SAME', activation='relu',
#                     name='Deconv3D_2'))

# model.add(Conv3D(128, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=128, padding='SAME', activation='relu',
#                  name='conv3D__3__1'))

# model.add(Conv3D(128, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=128, padding='SAME', activation='relu',
#                  name='conv3D__3__2'))
# model.add(BatchNormalization())

# # Layer4

# model.add(
#     Conv3DTranspose(64, kernel_size=(1, 3, 3), strides=(1, 2, 2), use_bias=64, padding='SAME', activation='relu',
#                     name='Deconv3D_3'))

# model.add(Conv3D(64, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=64, padding='SAME', activation='relu',
#                  name='conv3D__4__1'))

# model.add(Conv3D(64, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=64, padding='SAME', activation='relu',
#                      name='conv3D__4__2'))


# model.add(BatchNormalization())
# # Layer 5

# model.add(
#     Conv3DTranspose(32, kernel_size=(1, 3, 3), strides=(1, 2, 2), use_bias=32, padding='SAME', activation='relu',
#                     name='Deconv3D_4'))

# model.add(Conv3D(32, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=32, padding='SAME', activation='relu',
#                  name='conv3D__5__1'))

# model.add(Conv3D(16, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=16, padding='SAME', activation='relu',
#                  name='conv3D__5__2'))

# #model.add(Conv3D(8, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=8, padding='SAME', activation='relu',
#                  #name='conv3D2_23'))

# model.add(Conv3D(1, kernel_size=(1, 3, 3), strides=(1, 1, 1), use_bias=1, padding='SAME', activation='sigmoid',
#                                      name='conv3D__5__3'))
# #model.add(BatchNormalization())

Pompom

As it’s currently written, your answer is unclear. Please edit to add additional details that will help others understand how this addresses the question asked. You can find more information on how to write good answers in the help center. – Viewfinder